09:09:21.050 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:09:21.050 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4gMQ0A/test-tmp
09:09:21.050 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:09:21.050 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4gMQ0A/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:09:21.051 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
09:09:21.337 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:09:21.337 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:09:21.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.341063 1991 mini_oidc.cc:150] Starting JWKS server
09:09:21.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.344487 1991 webserver.cc:458] Webserver started at http://127.0.0.1:35315/ using document root <none> and password file <none>
09:09:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.344870 1991 mini_oidc.cc:186] Starting OIDC Discovery server
09:09:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.346392 1991 webserver.cc:458] Webserver started at http://127.0.0.1:43637/ using document root <none> and password file <none>
09:09:21.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:09:21Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:09:21.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:09:21Z Disabled control of system clock
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.402988 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/logs
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data/info.pb
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.254:32801
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.254
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json with env {}
09:09:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.671305 2003 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.671808 2003 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:21.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.672261 2003 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.700619 2003 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.700852 2003 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.701030 2003 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.701195 2003 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.701359 2003 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.701480 2003 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35315/jwks.json
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.732563 2003 master_runner.cc:386] Master server non-default flags:
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.254:32801
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data/info.pb
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.254
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/logs
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.733808 2003 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.735294 2003 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:21.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.749426 2009 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.750007 2010 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:21.751513 2003 server_base.cc:1034] running on GCE node
09:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:21.750671 2012 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.807241 2003 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:22.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.809980 2003 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.811369 2003 hybrid_clock.cc:648] HybridClock initialized: now 1736845762811335 us; error 55 us; skew 500 ppm
09:09:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.812064 2003 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.823173 2003 webserver.cc:458] Webserver started at http://127.1.241.254:45811/ using document root <none> and password file <none>
09:09:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.823971 2003 fs_manager.cc:362] Metadata directory not provided
09:09:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.824183 2003 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:22.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.824648 2003 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.829195 2003 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data/instance:
09:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451bebec617745b3b399cb84c8fca6a5"
09:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:22 on dist-test-slave-bm4z"
09:09:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.830092 2003 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal/instance:
09:09:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451bebec617745b3b399cb84c8fca6a5"
09:09:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:22 on dist-test-slave-bm4z"
09:09:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.836689 2003 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
09:09:22.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.841421 2019 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.842268 2003 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
09:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.842550 2003 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "451bebec617745b3b399cb84c8fca6a5"
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:22 on dist-test-slave-bm4z"
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.842841 2003 fs_report.cc:389] FS layout report
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data/data
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.890074 2003 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.891311 2003 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.891703 2003 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.954924 2003 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.254:32801
09:09:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.954994 2071 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.254:32801 every 8 connection(s)
09:09:22.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.957598 2003 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/data/info.pb
09:09:22.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.962232 2072 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.966464 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2003
09:09:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.966979 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-0/wal/instance
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.971815 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/logs
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data/info.pb
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.253:44947
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.253
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:22.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json with env {}
09:09:22.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.976461 2072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } has no permanent_uuid. Determining permanent_uuid...
09:09:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:22.997535 2072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } has no permanent_uuid. Determining permanent_uuid...
09:09:22.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:22.999488 2022 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.002916 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.052417 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 1
09:09:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.056071 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.124521 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 2
09:09:23.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.128254 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.219725 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 3
09:09:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.223832 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.251900 2074 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.252388 2074 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.252866 2074 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:23.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.279747 2074 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.279970 2074 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.280196 2074 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.280373 2074 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.280532 2074 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.280658 2074 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35315/jwks.json
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.310665 2074 master_runner.cc:386] Master server non-default flags:
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.253:44947
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data/info.pb
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.253
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/logs
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.311817 2074 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:23.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.313266 2074 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:23.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.326457 2083 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:23.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.367264 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 4
09:09:23.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.370839 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.670388 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 5
09:09:23.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.674494 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.222115 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 6
09:09:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.227926 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.253:44947: Network error: Client connection negotiation failed: client connection to 127.1.241.253:44947: connect: Connection refused (error 111)
09:09:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.327105 2084 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:23.329308 2074 server_base.cc:1034] running on GCE node
09:09:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:23.327956 2086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.409106 2074 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.411798 2074 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.413180 2074 hybrid_clock.cc:648] HybridClock initialized: now 1736845764413140 us; error 53 us; skew 500 ppm
09:09:24.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.413908 2074 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.420169 2074 webserver.cc:458] Webserver started at http://127.1.241.253:37377/ using document root <none> and password file <none>
09:09:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.420987 2074 fs_manager.cc:362] Metadata directory not provided
09:09:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.421140 2074 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.421468 2074 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.425763 2074 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data/instance:
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ba3826e9814486394fd4ac79ad5a249"
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:24 on dist-test-slave-bm4z"
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.426760 2074 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal/instance:
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ba3826e9814486394fd4ac79ad5a249"
09:09:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:24 on dist-test-slave-bm4z"
09:09:24.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.432953 2074 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.005s
09:09:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.437870 2095 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:24.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.438673 2074 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.438949 2074 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ba3826e9814486394fd4ac79ad5a249"
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:24 on dist-test-slave-bm4z"
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.439222 2074 fs_report.cc:389] FS layout report
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal
09:09:24.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data/data
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:24.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:24.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.487773 2074 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.489077 2074 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.489466 2074 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:24.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.553082 2074 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.253:44947
09:09:24.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.553171 2146 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.253:44947 every 8 connection(s)
09:09:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.555753 2074 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/data/info.pb
09:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.558674 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2074
09:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.559101 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-1/wal/instance
09:09:24.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.561303 2147 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.561983 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/logs
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data/info.pb
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.252:46849
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.252
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:24.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json with env {}
09:09:24.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.573870 2147 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } has no permanent_uuid. Determining permanent_uuid...
09:09:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.592936 2147 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } has no permanent_uuid. Determining permanent_uuid...
09:09:24.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.605777 2147 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } has no permanent_uuid. Determining permanent_uuid...
09:09:24.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.607923 2096 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:24.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.611629 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:24.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.661047 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 1
09:09:24.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.665189 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.733610 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 2
09:09:24.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.737488 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.828804 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 3
09:09:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.832362 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.843752 2149 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.844262 2149 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.844720 2149 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:24.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.871695 2149 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.871934 2149 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.872152 2149 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.872339 2149 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.872509 2149 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.872646 2149 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35315/jwks.json
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.902304 2149 master_runner.cc:386] Master server non-default flags:
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.252:46849
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35315/jwks.json
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:24.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data/info.pb
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.252
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/logs
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:24.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.903409 2149 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:24.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.904780 2149 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.917765 2160 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.975704 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 4
09:09:24.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.979293 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.278628 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 5
09:09:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.281937 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.288312 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } attempt: 7
09:09:25.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.298485 2072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } has no permanent_uuid. Determining permanent_uuid...
09:09:25.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.303552 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.362121 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 1
09:09:25.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.365800 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.447312 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 2
09:09:25.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.450848 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.536341 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 3
09:09:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.540091 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.680639 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 4
09:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.684341 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.829484 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 6
09:09:25.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.835345 2147 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.967845 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 5
09:09:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:25.971335 2072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.241.252:46849: Network error: Client connection negotiation failed: client connection to 127.1.241.252:46849: connect: Connection refused (error 111)
09:09:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.917788 2159 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:24.918032 2162 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:25.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:24.919426 2149 server_base.cc:1034] running on GCE node
09:09:25.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:25.999154 2149 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.001515 2149 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.002808 2149 hybrid_clock.cc:648] HybridClock initialized: now 1736845766002773 us; error 50 us; skew 500 ppm
09:09:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.003460 2149 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.008842 2149 webserver.cc:458] Webserver started at http://127.1.241.252:42861/ using document root <none> and password file <none>
09:09:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.009578 2149 fs_manager.cc:362] Metadata directory not provided
09:09:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.009737 2149 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.010067 2149 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.014326 2149 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data/instance:
09:09:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9db768cd20e84157900f1ea42d630f3a"
09:09:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:26 on dist-test-slave-bm4z"
09:09:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.015300 2149 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal/instance:
09:09:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9db768cd20e84157900f1ea42d630f3a"
09:09:26.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:26 on dist-test-slave-bm4z"
09:09:26.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.021531 2149 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
09:09:26.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.026263 2174 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.027083 2149 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.027330 2149 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9db768cd20e84157900f1ea42d630f3a"
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:26 on dist-test-slave-bm4z"
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.027603 2149 fs_report.cc:389] FS layout report
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal
09:09:26.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data/data
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:26.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.081341 2149 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.082754 2149 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:26.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.083151 2149 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:26.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.146656 2149 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.252:46849
09:09:26.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.146732 2226 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.252:46849 every 8 connection(s)
09:09:26.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.149410 2149 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/data/info.pb
09:09:26.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.153973 2227 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.157120 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2149
09:09:26.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.157565 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/master-2/wal/instance
09:09:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.170337 2227 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } has no permanent_uuid. Determining permanent_uuid...
09:09:26.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.191313 2227 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } has no permanent_uuid. Determining permanent_uuid...
09:09:26.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.199255 2227 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } has no permanent_uuid. Determining permanent_uuid...
09:09:26.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.212764 2227 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: Bootstrap starting.
09:09:26.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.217927 2227 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: Neither blocks nor log segments found. Creating new log.
09:09:26.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.219667 2227 log.cc:826] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: Log is configured to *not* fsync() on all Append() calls
09:09:26.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.223155 2227 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: No bootstrap required, opened a new log
09:09:26.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.241154 2227 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.241647 2227 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.241804 2227 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9db768cd20e84157900f1ea42d630f3a, State: Initialized, Role: FOLLOWER
09:09:26.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.242358 2227 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.244299 2236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.245255 2236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: This master's current role is: FOLLOWER
09:09:26.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.246294 2227 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: configured and running, proceeding with master startup.
09:09:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.267256 2247 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:09:26.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.267580 2247 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:09:26.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.262866 2149 master_runner.cc:186] Error getting master registration for 127.1.241.254:32801: OK, instance_id { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" instance_seqno: 1736845762936504 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:09:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.284919 2149 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.337874 2180 tablet.cc:2367] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:09:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.458801 2236 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.459291 2236 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.461447 2236 leader_election.cc:290] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 451bebec617745b3b399cb84c8fca6a5 (127.1.241.254:32801), 1ba3826e9814486394fd4ac79ad5a249 (127.1.241.253:44947)
09:09:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.523664 2072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 6
09:09:26.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.540356 2072 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: Bootstrap starting.
09:09:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.548209 2072 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: Neither blocks nor log segments found. Creating new log.
09:09:26.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.549693 2072 log.cc:826] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: Log is configured to *not* fsync() on all Append() calls
09:09:26.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.554087 2072 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: No bootstrap required, opened a new log
09:09:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.584276 2072 raft_consensus.cc:357] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.585116 2072 raft_consensus.cc:383] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:26.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.585412 2072 raft_consensus.cc:738] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 451bebec617745b3b399cb84c8fca6a5, State: Initialized, Role: FOLLOWER
09:09:26.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.586272 2072 consensus_queue.cc:260] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.589576 2251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.590284 2251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: This master's current role is: FOLLOWER
09:09:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.590857 2072 sys_catalog.cc:564] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: configured and running, proceeding with master startup.
09:09:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.591009 2047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9db768cd20e84157900f1ea42d630f3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "451bebec617745b3b399cb84c8fca6a5" is_pre_election: true
09:09:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.591733 2047 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9db768cd20e84157900f1ea42d630f3a in term 0.
09:09:26.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.592999 2175 leader_election.cc:304] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 451bebec617745b3b399cb84c8fca6a5, 9db768cd20e84157900f1ea42d630f3a; no voters:
09:09:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.594013 2236 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Leader pre-election won for term 1
09:09:26.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.594362 2236 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:09:26.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.594708 2236 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 0 FOLLOWER]: Advancing to term 1
09:09:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.607282 2236 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.610111 2047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9db768cd20e84157900f1ea42d630f3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "451bebec617745b3b399cb84c8fca6a5"
09:09:26.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.610566 2047 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 0 FOLLOWER]: Advancing to term 1
09:09:26.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.612408 2236 leader_election.cc:290] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [CANDIDATE]: Term 1 election: Requested vote from peers 451bebec617745b3b399cb84c8fca6a5 (127.1.241.254:32801), 1ba3826e9814486394fd4ac79ad5a249 (127.1.241.253:44947)
09:09:26.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.616638 2047 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9db768cd20e84157900f1ea42d630f3a in term 1.
09:09:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.617691 2175 leader_election.cc:304] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 451bebec617745b3b399cb84c8fca6a5, 9db768cd20e84157900f1ea42d630f3a; no voters:
09:09:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.618525 2236 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 1 FOLLOWER]: Leader election won for term 1
09:09:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.620254 2236 raft_consensus.cc:695] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [term 1 LEADER]: Becoming Leader. State: Replica: 9db768cd20e84157900f1ea42d630f3a, State: Running, Role: LEADER
09:09:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.621019 2236 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:26.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.640942 2263 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9db768cd20e84157900f1ea42d630f3a. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.641783 2263 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: This master's current role is: LEADER
09:09:26.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.649293 2025 tablet.cc:2367] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:09:26.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.650171 2266 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:09:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.656780 2266 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:09:26.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.656677 2262 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:09:26.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:26.658102 2262 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:09:26.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.626600 2003 master_runner.cc:186] Error getting master registration for 127.1.241.253:44947: OK, instance_id { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" instance_seqno: 1736845764535384 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:09:26.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.672379 2003 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:26.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.683488 2047 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [term 1 FOLLOWER]: Refusing update from remote peer 9db768cd20e84157900f1ea42d630f3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:09:26.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.685016 2263 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:09:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.718116 2251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9db768cd20e84157900f1ea42d630f3a. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.718778 2251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: This master's current role is: FOLLOWER
09:09:26.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.732954 2263 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.733600 2263 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: This master's current role is: LEADER
09:09:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.734195 2266 catalog_manager.cc:1348] Generated new cluster ID: d926dca39c2d4760acbd1e15b4d1c84e
09:09:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.734457 2266 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:09:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.738554 2251 sys_catalog.cc:455] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:26.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.739167 2251 sys_catalog.cc:458] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5 [sys.catalog]: This master's current role is: FOLLOWER
09:09:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.780580 2266 catalog_manager.cc:1371] Generated new certificate authority record
09:09:26.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.782369 2266 catalog_manager.cc:1505] Loading token signing keys...
09:09:26.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.804201 2266 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a: Generated new TSK 0
09:09:26.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.805941 2266 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:09:26.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.829617 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/logs
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data/info.pb
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.193:0
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.193
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.193
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:09:26.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.895776 2147 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } attempt: 7
09:09:26.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.943462 2147 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: Bootstrap starting.
09:09:26.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.960970 2147 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: Neither blocks nor log segments found. Creating new log.
09:09:26.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.965859 2147 log.cc:826] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: Log is configured to *not* fsync() on all Append() calls
09:09:26.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:26.993952 2147 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: No bootstrap required, opened a new log
09:09:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.064203 2147 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:27.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.066097 2147 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.067093 2147 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1ba3826e9814486394fd4ac79ad5a249, State: Initialized, Role: FOLLOWER
09:09:27.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.068888 2147 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.088841 2275 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.089502 2275 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: This master's current role is: FOLLOWER
09:09:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.089870 2275 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.090250 2275 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } }
09:09:27.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.092240 2275 leader_election.cc:290] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 451bebec617745b3b399cb84c8fca6a5 (127.1.241.254:32801), 9db768cd20e84157900f1ea42d630f3a (127.1.241.252:46849)
09:09:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.093274 2147 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: configured and running, proceeding with master startup.
09:09:27.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.093715 2120 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 0 FOLLOWER]: Advancing to term 1
09:09:27.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.096671 2122 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9db768cd20e84157900f1ea42d630f3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ba3826e9814486394fd4ac79ad5a249" is_pre_election: true
09:09:27.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.098927 2121 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9db768cd20e84157900f1ea42d630f3a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1ba3826e9814486394fd4ac79ad5a249"
09:09:27.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.105078 2047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ba3826e9814486394fd4ac79ad5a249" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "451bebec617745b3b399cb84c8fca6a5" is_pre_election: true
09:09:27.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.105964 2201 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1ba3826e9814486394fd4ac79ad5a249" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9db768cd20e84157900f1ea42d630f3a" is_pre_election: true
09:09:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.108016 2096 leader_election.cc:304] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1ba3826e9814486394fd4ac79ad5a249; no voters: 451bebec617745b3b399cb84c8fca6a5, 9db768cd20e84157900f1ea42d630f3a
09:09:27.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.109615 2120 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 1 FOLLOWER]: Refusing update from remote peer 9db768cd20e84157900f1ea42d630f3a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:09:27.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.111068 2236 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:09:27.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.168867 2276 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
09:09:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.174366 2275 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9db768cd20e84157900f1ea42d630f3a. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.175097 2275 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: This master's current role is: FOLLOWER
09:09:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.175381 2282 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:27.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.176162 2282 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249 [sys.catalog]: This master's current role is: FOLLOWER
09:09:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.234951 2292 catalog_manager.cc:1260] Loaded cluster ID: d926dca39c2d4760acbd1e15b4d1c84e
09:09:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.235208 2292 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: loading cluster ID for follower catalog manager: success
09:09:27.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.249472 2292 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: acquiring CA information for follower catalog manager: success
09:09:27.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.253931 2292 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 1ba3826e9814486394fd4ac79ad5a249: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.424741 2272 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.425513 2272 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.426417 2272 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.486783 2272 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.488147 2272 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.241.193
09:09:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.529680 2298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9db768cd20e84157900f1ea42d630f3a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "451bebec617745b3b399cb84c8fca6a5" member_type: VOTER last_known_addr { host: "127.1.241.254" port: 32801 } } peers { permanent_uuid: "1ba3826e9814486394fd4ac79ad5a249" member_type: VOTER last_known_addr { host: "127.1.241.253" port: 44947 } } peers { permanent_uuid: "9db768cd20e84157900f1ea42d630f3a" member_type: VOTER last_known_addr { host: "127.1.241.252" port: 46849 } } }
09:09:27.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.530347 2298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9db768cd20e84157900f1ea42d630f3a [sys.catalog]: This master's current role is: LEADER
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.552342 2272 tablet_server_runner.cc:78] Tablet server non-default flags:
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal
09:09:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.193:0
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data/info.pb
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.193
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.193
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/logs
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.555014 2272 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.557195 2272 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.604646 2302 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:27.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.663810 2262 catalog_manager.cc:1260] Loaded cluster ID: d926dca39c2d4760acbd1e15b4d1c84e
09:09:27.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.664309 2262 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: loading cluster ID for follower catalog manager: success
09:09:27.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.670509 2262 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: acquiring CA information for follower catalog manager: success
09:09:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.681031 2262 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 451bebec617745b3b399cb84c8fca6a5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.606995 2307 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:27.608410 2272 server_base.cc:1034] running on GCE node
09:09:28.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:27.605302 2303 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.926594 2272 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:28.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.929260 2272 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:28.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.930742 2272 hybrid_clock.cc:648] HybridClock initialized: now 1736845768930686 us; error 75 us; skew 500 ppm
09:09:28.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.931645 2272 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.943011 2272 webserver.cc:458] Webserver started at http://127.1.241.193:41901/ using document root <none> and password file <none>
09:09:28.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.943836 2272 fs_manager.cc:362] Metadata directory not provided
09:09:28.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.944070 2272 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:28.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.944494 2272 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.949020 2272 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data/instance:
09:09:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7e31a5a74ea47719d86f32273270ae0"
09:09:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:28 on dist-test-slave-bm4z"
09:09:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.949965 2272 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal/instance:
09:09:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7e31a5a74ea47719d86f32273270ae0"
09:09:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:28 on dist-test-slave-bm4z"
09:09:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.962860 2272 fs_manager.cc:696] Time spent creating directory manager: real 0.012s user 0.008s sys 0.005s
09:09:28.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.979066 2313 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:28.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.980664 2272 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.000s sys 0.014s
09:09:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.981071 2272 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal
09:09:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f7e31a5a74ea47719d86f32273270ae0"
09:09:28.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:28 on dist-test-slave-bm4z"
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:28.981534 2272 fs_report.cc:389] FS layout report
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data/data
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:28.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:29.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.033793 2272 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:29.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.035030 2272 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:29.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.035486 2272 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.038343 2272 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:09:29.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.042748 2272 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:09:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.042932 2272 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:09:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.043184 2272 ts_tablet_manager.cc:610] Registered 0 tablets
09:09:29.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.043331 2272 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:09:29.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.246762 2272 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.193:36753
09:09:29.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.246883 2425 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.193:36753 every 8 connection(s)
09:09:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.250152 2272 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/data/info.pb
09:09:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.250880 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2272
09:09:29.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.251282 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-0/wal/instance
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.268174 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/logs
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data/info.pb
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.194:0
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.194
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.194
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:09:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.339015 2426 heartbeater.cc:346] Connected to a master server at 127.1.241.252:46849
09:09:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.339385 2426 heartbeater.cc:463] Registering TS with master...
09:09:29.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.340515 2426 heartbeater.cc:510] Master 127.1.241.252:46849 requested a full tablet report, sending...
09:09:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.343626 2191 ts_manager.cc:194] Registered new tserver with Master: f7e31a5a74ea47719d86f32273270ae0 (127.1.241.193:36753)
09:09:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.344494 2428 heartbeater.cc:346] Connected to a master server at 127.1.241.253:44947
09:09:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.344799 2428 heartbeater.cc:463] Registering TS with master...
09:09:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.345608 2428 heartbeater.cc:510] Master 127.1.241.253:44947 requested a full tablet report, sending...
09:09:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.346107 2191 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.241.193:33113
09:09:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.349001 2112 ts_manager.cc:194] Registered new tserver with Master: f7e31a5a74ea47719d86f32273270ae0 (127.1.241.193:36753)
09:09:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.356199 2433 heartbeater.cc:346] Connected to a master server at 127.1.241.254:32801
09:09:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.356505 2433 heartbeater.cc:463] Registering TS with master...
09:09:29.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.357075 2433 heartbeater.cc:510] Master 127.1.241.254:32801 requested a full tablet report, sending...
09:09:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.359784 2037 ts_manager.cc:194] Registered new tserver with Master: f7e31a5a74ea47719d86f32273270ae0 (127.1.241.193:36753)
09:09:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.633477 2427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.633917 2427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.634358 2427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:29.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.663545 2427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:29.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.664464 2427 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.241.194
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.697165 2427 tablet_server_runner.cc:78] Tablet server non-default flags:
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.194:0
09:09:29.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data/info.pb
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.194
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.194
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/logs
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.698323 2427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.699753 2427 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.713784 2442 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:30.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.354812 2426 heartbeater.cc:502] Master 127.1.241.252:46849 was elected leader, sending a full tablet report...
09:09:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.714210 2443 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:29.716295 2427 server_base.cc:1034] running on GCE node
09:09:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:29.714541 2445 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:30.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.762621 2427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.764972 2427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.766394 2427 hybrid_clock.cc:648] HybridClock initialized: now 1736845770766353 us; error 63 us; skew 500 ppm
09:09:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.767135 2427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:30.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.773000 2427 webserver.cc:458] Webserver started at http://127.1.241.194:42331/ using document root <none> and password file <none>
09:09:30.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.773736 2427 fs_manager.cc:362] Metadata directory not provided
09:09:30.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.773917 2427 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:30.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.774262 2427 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:30.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.778885 2427 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data/instance:
09:09:30.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d41ffd8268746fe85f154b0f2e39445"
09:09:30.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:30 on dist-test-slave-bm4z"
09:09:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.779860 2427 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal/instance:
09:09:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d41ffd8268746fe85f154b0f2e39445"
09:09:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:30 on dist-test-slave-bm4z"
09:09:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.785717 2427 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
09:09:30.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.790557 2452 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.791352 2427 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
09:09:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.791637 2427 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal
09:09:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d41ffd8268746fe85f154b0f2e39445"
09:09:30.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:30 on dist-test-slave-bm4z"
09:09:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.791905 2427 fs_report.cc:389] FS layout report
09:09:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal
09:09:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal
09:09:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data/data
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.850551 2427 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.851755 2427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.852185 2427 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.854369 2427 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:09:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.858016 2427 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:09:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.858191 2427 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:09:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.858439 2427 ts_tablet_manager.cc:610] Registered 0 tablets
09:09:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.858579 2427 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:09:30.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.984704 2427 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.194:41761
09:09:30.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.984808 2565 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.194:41761 every 8 connection(s)
09:09:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.987174 2427 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/data/info.pb
09:09:30.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.990128 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2427
09:09:30.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:30.990549 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-1/wal/instance
09:09:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.005437 1991 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/logs
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data/info.pb
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.195:0
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.195
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.195
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:09:31.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.022330 2566 heartbeater.cc:346] Connected to a master server at 127.1.241.252:46849
09:09:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.022811 2566 heartbeater.cc:463] Registering TS with master...
09:09:31.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.024114 2566 heartbeater.cc:510] Master 127.1.241.252:46849 requested a full tablet report, sending...
09:09:31.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.027386 2191 ts_manager.cc:194] Registered new tserver with Master: 0d41ffd8268746fe85f154b0f2e39445 (127.1.241.194:41761)
09:09:31.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.029147 2191 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.241.194:45829
09:09:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.030483 2567 heartbeater.cc:346] Connected to a master server at 127.1.241.253:44947
09:09:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.030800 2567 heartbeater.cc:463] Registering TS with master...
09:09:31.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.033635 2567 heartbeater.cc:510] Master 127.1.241.253:44947 requested a full tablet report, sending...
09:09:31.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.036202 2112 ts_manager.cc:194] Registered new tserver with Master: 0d41ffd8268746fe85f154b0f2e39445 (127.1.241.194:41761)
09:09:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.042915 2568 heartbeater.cc:346] Connected to a master server at 127.1.241.254:32801
09:09:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.043159 2568 heartbeater.cc:463] Registering TS with master...
09:09:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.043586 2568 heartbeater.cc:510] Master 127.1.241.254:32801 requested a full tablet report, sending...
09:09:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.045632 2037 ts_manager.cc:194] Registered new tserver with Master: 0d41ffd8268746fe85f154b0f2e39445 (127.1.241.194:41761)
09:09:31.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.283664 2573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:31.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.284045 2573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:31.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.284479 2573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.314006 2573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.314682 2573 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.241.195
09:09:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.343966 2573 tablet_server_runner.cc:78] Tablet server non-default flags:
09:09:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.241.212:33713
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.241.195:0
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data/info.pb
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.241.195
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.241.254:32801,127.1.241.253:44947,127.1.241.252:46849
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.241.195
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/logs
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.345078 2573 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:31.346558 2573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:31.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.361928 2582 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:32.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.033484 2566 heartbeater.cc:502] Master 127.1.241.252:46849 was elected leader, sending a full tablet report...
09:09:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.362136 2583 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:31.362735 2585 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:32.430596 2584 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:09:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.430712 2573 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.434155 2573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.436262 2573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.437606 2573 hybrid_clock.cc:648] HybridClock initialized: now 1736845772437560 us; error 48 us; skew 500 ppm
09:09:32.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.438340 2573 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.443784 2573 webserver.cc:458] Webserver started at http://127.1.241.195:32795/ using document root <none> and password file <none>
09:09:32.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.444582 2573 fs_manager.cc:362] Metadata directory not provided
09:09:32.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.444787 2573 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:32.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.445187 2573 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.449541 2573 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data/instance:
09:09:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a308555baf44e7ea588105ef7789cac"
09:09:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:32 on dist-test-slave-bm4z"
09:09:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.450493 2573 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal/instance:
09:09:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a308555baf44e7ea588105ef7789cac"
09:09:32.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:32 on dist-test-slave-bm4z"
09:09:32.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.456810 2573 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
09:09:32.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.461773 2594 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.462575 2573 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.462850 2573 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0a308555baf44e7ea588105ef7789cac"
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:32 on dist-test-slave-bm4z"
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.463127 2573 fs_report.cc:389] FS layout report
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data/data
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:32.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.510366 2573 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:32.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.511605 2573 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:32.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.512079 2573 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:32.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.514304 2573 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:09:32.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.518348 2573 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:09:32.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.518548 2573 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
09:09:32.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.518831 2573 ts_tablet_manager.cc:610] Registered 0 tablets
09:09:32.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.519001 2573 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
09:09:32.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.647421 2573 rpc_server.cc:307] RPC server started. Bound to: 127.1.241.195:37183
09:09:32.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.647516 2707 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.241.195:37183 every 8 connection(s)
09:09:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.649698 2573 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/data/info.pb
09:09:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.656668 1991 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2573
09:09:32.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.657392 1991 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster719876072983527509/ts-2/wal/instance
09:09:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.699623 2708 heartbeater.cc:346] Connected to a master server at 127.1.241.252:46849
09:09:32.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.700078 2708 heartbeater.cc:463] Registering TS with master...
09:09:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.701154 2709 heartbeater.cc:346] Connected to a master server at 127.1.241.253:44947
09:09:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.701224 2708 heartbeater.cc:510] Master 127.1.241.252:46849 requested a full tablet report, sending...
09:09:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.701464 2709 heartbeater.cc:463] Registering TS with master...
09:09:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.702236 2709 heartbeater.cc:510] Master 127.1.241.253:44947 requested a full tablet report, sending...
09:09:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.702827 2710 heartbeater.cc:346] Connected to a master server at 127.1.241.254:32801
09:09:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.703115 2710 heartbeater.cc:463] Registering TS with master...
09:09:32.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.703362 2191 ts_manager.cc:194] Registered new tserver with Master: 0a308555baf44e7ea588105ef7789cac (127.1.241.195:37183)
09:09:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.703827 2710 heartbeater.cc:510] Master 127.1.241.254:32801 requested a full tablet report, sending...
09:09:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.704612 2111 ts_manager.cc:194] Registered new tserver with Master: 0a308555baf44e7ea588105ef7789cac (127.1.241.195:37183)
09:09:32.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.704823 2191 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.241.195:37653
09:09:32.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.705955 2037 ts_manager.cc:194] Registered new tserver with Master: 0a308555baf44e7ea588105ef7789cac (127.1.241.195:37183)
09:09:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:32.720732 1991 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
09:09:32.730 [DEBUG - main] (MiniKuduCluster.java:195) Response:
09:09:32.730 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
09:09:32.730 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.241.254"
port: 32801
}
bound_http_address {
host: "127.1.241.254"
port: 45811
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.241.253"
port: 44947
}
bound_http_address {
host: "127.1.241.253"
port: 37377
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.241.252"
port: 46849
}
bound_http_address {
host: "127.1.241.252"
port: 42861
}
}
}
09:09:32.731 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:09:32.731 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.241.193"
port: 36753
}
bound_http_address {
host: "127.1.241.193"
port: 41901
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.241.194"
port: 41761
}
bound_http_address {
host: "127.1.241.194"
port: 42331
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.241.195"
port: 37183
}
bound_http_address {
host: "127.1.241.195"
port: 32795
}
}
}
09:09:32.731 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:09:32.732 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster7450162716955309983/krb5kdc/krb5cc
09:09:32.733 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:09:32.733 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4gMQ0A/test-tmp
09:09:32.734 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:09:32.734 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4gMQ0A/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:09:32.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
09:09:33.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:33.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
09:09:33.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:33.029678 2719 mini_oidc.cc:150] Starting JWKS server
09:09:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:33.032497 2719 webserver.cc:458] Webserver started at http://127.0.0.1:33975/ using document root <none> and password file <none>
09:09:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:33.032835 2719 mini_oidc.cc:186] Starting OIDC Discovery server
09:09:33.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:33.034205 2719 webserver.cc:458] Webserver started at http://127.0.0.1:46067/ using document root <none> and password file <none>
09:09:33.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
09:09:33.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/principal' for realm 'KRBTEST.COM',
09:09:33.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
09:09:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:33 dist-test-slave-bm4z krb5kdc[2730](info): setting up network...
09:09:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
09:09:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:33 dist-test-slave-bm4z krb5kdc[2730](info): set up 2 sockets
09:09:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:33 dist-test-slave-bm4z krb5kdc[2730](info): commencing operation
09:09:33.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
09:09:33.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:33.707561 2708 heartbeater.cc:502] Master 127.1.241.252:46849 was elected leader, sending a full tablet report...
09:09:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.027549 2719 mini_kdc.cc:121] Time spent starting KDC: real 0.955s user 0.012s sys 0.004s
09:09:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
09:09:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:09:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
09:09:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
09:09:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:09:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
09:09:34.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
09:09:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:09:34.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
09:09:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:09:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/test-user.keytab.
09:09:34.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/test-user.keytab.
09:09:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:34 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:34.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:09:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:09:34Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:09:34.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:09:34Z Disabled control of system clock
09:09:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.167.254@KRBTEST.COM; defaulting to no policy
09:09:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.167.254@KRBTEST.COM" created.
09:09:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:34.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab.
09:09:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab.
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:34.183717 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.448815 2744 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.449282 2744 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.449877 2744 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.477574 2744 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.477772 2744 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.477993 2744 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.478160 2744 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.478355 2744 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.478524 2744 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.478652 2744 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:34.508756 2744 master_runner.cc:386] Master server non-default flags:
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:34.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:34.509785 2744 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:34.510633 2744 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.523919 2750 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.524441 2751 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:34.524448 2753 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:35.592305 2752 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1069 milliseconds
09:09:35.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.592417 2744 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:35.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.596017 2744 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.598368 2744 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.599660 2744 hybrid_clock.cc:648] HybridClock initialized: now 1736845775599627 us; error 50 us; skew 500 ppm
09:09:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.600435 2744 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:35 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845775, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.611837 2744 init.cc:377] Logged in from keytab as kudu/127.2.167.254@KRBTEST.COM (short username kudu)
09:09:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.616519 2744 webserver.cc:458] Webserver started at http://127.2.167.254:33327/ using document root <none> and password file <none>
09:09:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.617372 2744 fs_manager.cc:362] Metadata directory not provided
09:09:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.617547 2744 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:35.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.617895 2744 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.622241 2744 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/instance:
09:09:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.623170 2744 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal/instance:
09:09:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:35.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.629570 2744 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
09:09:35.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.634142 2764 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.635035 2744 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:09:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.635306 2744 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.635685 2744 fs_report.cc:389] FS layout report
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/data
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:35.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:35.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.694470 2744 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.699362 2744 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:35.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.699699 2744 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.762275 2744 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.254:33565
09:09:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.762342 2815 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.254:33565 every 8 connection(s)
09:09:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.764750 2744 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.769583 2816 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.773818 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2744
09:09:35.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.774222 2719 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal/instance
09:09:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.780129 2816 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } has no permanent_uuid. Determining permanent_uuid...
09:09:35.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.167.253@KRBTEST.COM; defaulting to no policy
09:09:35.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.167.253@KRBTEST.COM" created.
09:09:35.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:35 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845775, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab.
09:09:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab.
09:09:35.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.812150 2816 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } has no permanent_uuid. Determining permanent_uuid...
09:09:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.813656 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:35.814011 2767 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:35.818038 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:35.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.867406 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 1
09:09:35.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:35.870606 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:35.938911 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 2
09:09:35.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:35.942165 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.033533 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 3
09:09:36.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.036805 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:36.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.102269 2823 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.102854 2823 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:36.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.103544 2823 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:36.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.135654 2823 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:36.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.135859 2823 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.136173 2823 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.136379 2823 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.136631 2823 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.136848 2823 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:36.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.136982 2823 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.180222 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 4
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.183218 2823 master_runner.cc:386] Master server non-default flags:
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.183565 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.184725 2823 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.185782 2823 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:36.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.199623 2829 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.482952 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 5
09:09:36.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.486407 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:37.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.034010 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 6
09:09:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.040589 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.253:39481: Network error: Client connection negotiation failed: client connection to 127.2.167.253:39481: connect: Connection refused (error 111)
09:09:37.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.200102 2830 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:37.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:36.201057 2823 server_base.cc:1034] running on GCE node
09:09:37.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:36.200512 2832 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:37.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.307469 2823 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.310179 2823 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.311508 2823 hybrid_clock.cc:648] HybridClock initialized: now 1736845777311474 us; error 39 us; skew 500 ppm
09:09:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.312368 2823 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:37 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845777, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.318090 2823 init.cc:377] Logged in from keytab as kudu/127.2.167.253@KRBTEST.COM (short username kudu)
09:09:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.323312 2823 webserver.cc:458] Webserver started at http://127.2.167.253:39041/ using document root <none> and password file <none>
09:09:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.324182 2823 fs_manager.cc:362] Metadata directory not provided
09:09:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.324359 2823 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:37.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.324738 2823 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.329434 2823 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/instance:
09:09:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.330427 2823 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal/instance:
09:09:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:37.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.337090 2823 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
09:09:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.342007 2842 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.342906 2823 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
09:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.343212 2823 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:37.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.343506 2823 fs_report.cc:389] FS layout report
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/data
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:37.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:37.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.412498 2823 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.417636 2823 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.418061 2823 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.482040 2823 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.253:39481
09:09:37.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.482120 2895 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.253:39481 every 8 connection(s)
09:09:37.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.484344 2823 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.485828 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2823
09:09:37.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.486388 2719 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal/instance
09:09:37.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.489739 2896 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:37.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.167.252@KRBTEST.COM; defaulting to no policy
09:09:37.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.500058 2896 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } has no permanent_uuid. Determining permanent_uuid...
09:09:37.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:37.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.167.252@KRBTEST.COM" created.
09:09:37.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:37 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845777, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:09:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab.
09:09:37.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.167.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab.
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.523952 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:37.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:37.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.530645 2896 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } has no permanent_uuid. Determining permanent_uuid...
09:09:37.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:37 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845777, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.555342 2896 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } has no permanent_uuid. Determining permanent_uuid...
09:09:37.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.557437 2844 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.561239 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:37.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.610641 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 1
09:09:37.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.614068 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:37.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.682552 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 2
09:09:37.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.686329 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.777712 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 3
09:09:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.781597 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.809407 2902 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.809841 2902 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.810317 2902 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.837239 2902 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.837409 2902 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.837643 2902 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:37.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.837812 2902 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.837973 2902 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.838126 2902 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:37.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.838244 2902 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.868868 2902 master_runner.cc:386] Master server non-default flags:
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.870074 2902 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:37.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.870945 2902 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:37.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.884570 2911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:37.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:37.924933 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 4
09:09:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.928418 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.100941 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } attempt: 7
09:09:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:38 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845775, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:38.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.143113 2816 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } has no permanent_uuid. Determining permanent_uuid...
09:09:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.173539 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.227778 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 5
09:09:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.232187 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 1
09:09:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.233256 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.237866 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.319341 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 2
09:09:38.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.325202 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.410684 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 3
09:09:38.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.417644 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.558110 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 4
09:09:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.571733 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.780777 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 6
09:09:38.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.791030 2896 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:38.855602 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 5
09:09:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:38.860551 2816 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.167.252:37057: Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.884642 2910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:37.884856 2913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.034399 2912 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1147 milliseconds
09:09:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.034504 2902 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:39.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.035631 2902 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:39.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.037947 2902 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.039234 2902 hybrid_clock.cc:648] HybridClock initialized: now 1736845779039207 us; error 41 us; skew 500 ppm
09:09:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.039886 2902 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.044970 2902 init.cc:377] Logged in from keytab as kudu/127.2.167.252@KRBTEST.COM (short username kudu)
09:09:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.049539 2902 webserver.cc:458] Webserver started at http://127.2.167.252:41465/ using document root <none> and password file <none>
09:09:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.050271 2902 fs_manager.cc:362] Metadata directory not provided
09:09:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.050424 2902 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:39.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.050731 2902 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
09:09:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.055245 2902 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/instance:
09:09:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.056299 2902 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal/instance:
09:09:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.063383 2902 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
09:09:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.068154 2927 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.069257 2902 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
09:09:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.069628 2902 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.069985 2902 fs_report.cc:389] FS layout report
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/data
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:39.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.128957 2902 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.133388 2902 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.133741 2902 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.203763 2902 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.252:37057
09:09:39.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.203846 2978 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.252:37057 every 8 connection(s)
09:09:39.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.206084 2902 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.211051 2979 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:09:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.212024 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 2902
09:09:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.212363 2719 external_mini_cluster.cc:1427] Reading /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal/instance
09:09:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.225844 2979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } has no permanent_uuid. Determining permanent_uuid...
09:09:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.252@KRBTEST.COM
09:09:39.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:39.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.257462 2979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } has no permanent_uuid. Determining permanent_uuid...
09:09:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:39.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.278019 2979 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } has no permanent_uuid. Determining permanent_uuid...
09:09:39.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for kudu/127.2.167.252@KRBTEST.COM
09:09:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845774, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.301324 2979 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap starting.
09:09:39.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.302814 2719 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
09:09:39.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.306146 2979 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Neither blocks nor log segments found. Creating new log.
09:09:39.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.307633 2979 log.cc:826] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Log is configured to *not* fsync() on all Append() calls
09:09:39.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:39.309 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
09:09:39.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
09:09:39.310 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
09:09:39.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.167.254"
port: 33565
}
bound_http_address {
host: "127.2.167.254"
port: 33327
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.167.253"
port: 39481
}
bound_http_address {
host: "127.2.167.253"
port: 39041
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.167.252"
port: 37057
}
bound_http_address {
host: "127.2.167.252"
port: 41465
}
}
}
09:09:39.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
09:09:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.311774 2979 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: No bootstrap required, opened a new log
09:09:39.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:39.312 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:09:39.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
09:09:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.329018 2979 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.329512 2979 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.329726 2979 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Initialized, Role: FOLLOWER
09:09:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.330356 2979 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:39.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
09:09:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.332779 2990 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.333510 2990 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:39.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.334264 2979 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: configured and running, proceeding with master startup.
09:09:39.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:39.337 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:39.338 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:09:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.354425 3001 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:09:39.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.354630 3001 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:09:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.352154 2902 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: OK, instance_id { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" instance_seqno: 1736845775744323 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:09:39.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.381245 2902 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:39.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.388062 2933 tablet.cc:2367] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:09:39.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.412962 2816 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 6
09:09:39.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845775, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for kudu/127.2.167.252@KRBTEST.COM
09:09:39.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.435120 2816 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap starting.
09:09:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.441649 2816 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Neither blocks nor log segments found. Creating new log.
09:09:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.443701 2816 log.cc:826] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Log is configured to *not* fsync() on all Append() calls
09:09:39.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.447540 2816 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: No bootstrap required, opened a new log
09:09:39.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.464099 2816 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.464589 2816 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.464818 2816 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3130793f52ba4f65a4b1e245415a26b9, State: Initialized, Role: FOLLOWER
09:09:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.465441 2816 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.467862 3005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.468714 3005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.468745 2990 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.469172 2990 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.469628 2816 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: configured and running, proceeding with master startup.
09:09:39.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.471369 2990 leader_election.cc:290] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 30ebf93cb2f746f297dda1a92ead8278 (127.2.167.253:39481)
09:09:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.473753 2791 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3130793f52ba4f65a4b1e245415a26b9" is_pre_election: true
09:09:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.474566 2791 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6356baec59b549689c7fa334e38f4a09 in term 0.
09:09:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.476241 2930 leader_election.cc:304] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3130793f52ba4f65a4b1e245415a26b9, 6356baec59b549689c7fa334e38f4a09; no voters:
09:09:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.477268 2990 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:09:39.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.477614 2990 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:09:39.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.477939 2990 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 0 FOLLOWER]: Advancing to term 1
09:09:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.485201 2990 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.487859 2791 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.488410 2791 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 0 FOLLOWER]: Advancing to term 1
09:09:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.496091 2791 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6356baec59b549689c7fa334e38f4a09 in term 1.
09:09:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.498229 2990 leader_election.cc:290] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 1 election: Requested vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 30ebf93cb2f746f297dda1a92ead8278 (127.2.167.253:39481)
09:09:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.501225 2930 leader_election.cc:304] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3130793f52ba4f65a4b1e245415a26b9, 6356baec59b549689c7fa334e38f4a09; no voters:
09:09:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.503401 2990 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Leader election won for term 1
09:09:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.517505 2990 raft_consensus.cc:695] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 LEADER]: Becoming Leader. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Running, Role: LEADER
09:09:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.519182 2990 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.535435 3016 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.537250 3016 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: LEADER
09:09:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.543670 3018 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:09:39.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:39.544114 3018 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:09:39.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.545462 3021 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:09:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.551741 3021 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:09:39.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.524988 2744 master_runner.cc:186] Error getting master registration for 127.2.167.253:39481: OK, instance_id { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" instance_seqno: 1736845777464777 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:09:39.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.574903 2744 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.598706 2791 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 1 FOLLOWER]: Refusing update from remote peer 6356baec59b549689c7fa334e38f4a09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:09:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.600211 2990 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:09:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.632803 3005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.633494 3005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.634707 2990 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.635329 2990 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: LEADER
09:09:39.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.637707 3021 catalog_manager.cc:1348] Generated new cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:39.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.637944 3021 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:09:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.659909 3005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.660583 3005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.694303 3021 catalog_manager.cc:1371] Generated new certificate authority record
09:09:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.700066 3021 catalog_manager.cc:1505] Loading token signing keys...
09:09:39.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.739511 3021 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Generated new TSK 0
09:09:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.740993 3021 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:09:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.851492 2896 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } attempt: 7
09:09:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:39 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845777, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for kudu/127.2.167.252@KRBTEST.COM
09:09:39.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.885049 2896 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap starting.
09:09:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.892387 2896 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Neither blocks nor log segments found. Creating new log.
09:09:39.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.894348 2896 log.cc:826] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Log is configured to *not* fsync() on all Append() calls
09:09:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.899771 2896 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: No bootstrap required, opened a new log
09:09:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.928781 2896 raft_consensus.cc:357] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.929522 2896 raft_consensus.cc:383] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:09:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.929811 2896 raft_consensus.cc:738] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Initialized, Role: FOLLOWER
09:09:39.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.930761 2896 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.933202 3030 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.933955 3030 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.936479 2896 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: configured and running, proceeding with master startup.
09:09:39.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.936998 2869 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 0 FOLLOWER]: Advancing to term 1
09:09:39.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.940933 2870 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:39.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.943874 2871 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "30ebf93cb2f746f297dda1a92ead8278" is_pre_election: true
09:09:39.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.948822 2869 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Refusing update from remote peer 6356baec59b549689c7fa334e38f4a09: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
09:09:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:39.950984 3016 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
09:09:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.002151 3030 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.002808 3030 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:40.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.003762 3036 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:40.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.004642 3036 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:40.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.054607 3046 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:40.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.054865 3046 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: loading cluster ID for follower catalog manager: success
09:09:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.065049 3046 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: acquiring CA information for follower catalog manager: success
09:09:40.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.070055 3046 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:40.339 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.253:39481
09:09:40.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:09:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.341804 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2823
09:09:40.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:40.381 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.252:37057
09:09:40.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:09:40.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.381999 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2902
09:09:40.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:40.404 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.254:33565
09:09:40.404 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:09:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.405205 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2744
09:09:40.427 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:40.427 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.253:39481
09:09:40.428 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:09:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.429570 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:40.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:40.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.700066 3048 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:40.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.700546 3048 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:40.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.701073 3048 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.729064 3048 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.729252 3048 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.729487 3048 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.729673 3048 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.729848 3048 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.730022 3048 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.730142 3048 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.766268 3048 master_runner.cc:386] Master server non-default flags:
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:40.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:40.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.767640 3048 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:40.768525 3048 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.782413 3055 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.782657 3054 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:41.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:40.782495 3057 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:41.901083 3056 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:09:41.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.901244 3048 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:41.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.905158 3048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.907997 3048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.909384 3048 hybrid_clock.cc:648] HybridClock initialized: now 1736845781909348 us; error 53 us; skew 500 ppm
09:09:41.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.910115 3048 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:41.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:41 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845781, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:41.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.921501 3048 init.cc:377] Logged in from keytab as kudu/127.2.167.253@KRBTEST.COM (short username kudu)
09:09:41.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.926066 3048 webserver.cc:458] Webserver started at http://127.2.167.253:39041/ using document root <none> and password file <none>
09:09:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.926913 3048 fs_manager.cc:362] Metadata directory not provided
09:09:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.927098 3048 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:41.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.933898 3048 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.002s sys 0.003s
09:09:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.937886 3068 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:41.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.938751 3048 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.939019 3048 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.941063 3048 fs_report.cc:389] FS layout report
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/data
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:41.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:41.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.990923 3048 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.995726 3048 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:41.996172 3048 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.058681 3048 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.253:39481
09:09:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.058744 3119 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.253:39481 every 8 connection(s)
09:09:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.061103 3048 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.062044 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3048
09:09:42.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:42.062 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.252:37057
09:09:42.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.064222 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.072981 3120 sys_catalog.cc:263] Verifying existing consensus state
09:09:42.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.081897 3120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap starting.
09:09:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.128122 3120 log.cc:826] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Log is configured to *not* fsync() on all Append() calls
09:09:42.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.144814 3120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:42.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.145838 3120 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap complete.
09:09:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.181757 3120 raft_consensus.cc:357] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:42.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.185210 3120 raft_consensus.cc:738] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Initialized, Role: FOLLOWER
09:09:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.186091 3120 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.188871 3124 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:42.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.189587 3124 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.190989 3120 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: configured and running, proceeding with master startup.
09:09:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.219502 3135 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:42.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.219962 3135 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: loading cluster ID for follower catalog manager: success
09:09:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.229477 3135 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: acquiring CA information for follower catalog manager: success
09:09:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.232780 3135 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.230320 3048 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:42.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.258565 3048 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:42.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.406308 3122 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:42.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.406802 3122 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:42.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.407394 3122 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.434643 3122 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.434826 3122 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.435048 3122 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.435217 3122 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.435382 3122 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.435573 3122 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.435720 3122 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.468438 3122 master_runner.cc:386] Master server non-default flags:
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.469578 3122 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:42.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:42.470489 3122 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.485307 3143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.339763 3152 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:43.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.340541 3152 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:43.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.344688 3152 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:43.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:43.359186 3071 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:43.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:43.364701 3071 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:43.365693 3069 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057): Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.366060 3069 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278; no voters: 3130793f52ba4f65a4b1e245415a26b9, 6356baec59b549689c7fa334e38f4a09
09:09:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.366956 3152 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:09:43.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.485373 3144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:43.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:42.487421 3146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:43.620966 3145 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:09:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.621122 3122 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.622673 3122 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:43.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.625641 3122 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:43.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.627147 3122 hybrid_clock.cc:648] HybridClock initialized: now 1736845783627068 us; error 76 us; skew 500 ppm
09:09:43.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.628234 3122 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:43 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845783, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.634785 3122 init.cc:377] Logged in from keytab as kudu/127.2.167.252@KRBTEST.COM (short username kudu)
09:09:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.640221 3122 webserver.cc:458] Webserver started at http://127.2.167.252:41465/ using document root <none> and password file <none>
09:09:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.641045 3122 fs_manager.cc:362] Metadata directory not provided
09:09:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.641254 3122 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.649210 3122 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.008s sys 0.000s
09:09:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.653752 3161 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:43.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.654734 3122 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.655021 3122 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.657330 3122 fs_report.cc:389] FS layout report
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:43.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/data
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:43.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.720144 3122 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:43.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.725445 3122 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:43.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.725888 3122 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.807098 3122 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.252:37057
09:09:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.807181 3212 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.252:37057 every 8 connection(s)
09:09:43.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.809813 3122 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:43.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.811476 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3122
09:09:43.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:43.812 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.254:33565
09:09:43.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.814432 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:43.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:43.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:43.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.826028 3213 sys_catalog.cc:263] Verifying existing consensus state
09:09:43.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.832798 3213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap starting.
09:09:43.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.879901 3213 log.cc:826] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Log is configured to *not* fsync() on all Append() calls
09:09:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.897976 3213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.898816 3213 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap complete.
09:09:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.932487 3213 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:43.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.935349 3213 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Initialized, Role: FOLLOWER
09:09:43.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.936316 3213 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:43.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.939186 3217 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:43.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.940246 3217 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:43.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.940776 3213 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: configured and running, proceeding with master startup.
09:09:43.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.978240 3228 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:43.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.978482 3228 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: loading cluster ID for follower catalog manager: success
09:09:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.995766 3228 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: acquiring CA information for follower catalog manager: success
09:09:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.971892 3122 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:43.998324 3122 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:44.001055 3228 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.130577 3215 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:44.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.131049 3215 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:44.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.131536 3215 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.173532 3215 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.173826 3215 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.174242 3215 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.174525 3215 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.174818 3215 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.175127 3215 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.175352 3215 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:44.210916 3215 master_runner.cc:386] Master server non-default flags:
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:44.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:44.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:44.212025 3215 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:44.212960 3215 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.229074 3236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.328872 3243 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.329324 3243 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.331344 3243 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.372406 3188 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "30ebf93cb2f746f297dda1a92ead8278" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6356baec59b549689c7fa334e38f4a09" is_pre_election: true
09:09:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.373330 3188 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 30ebf93cb2f746f297dda1a92ead8278 in term 1.
09:09:45.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:45.373749 3071 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.374708 3069 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.375351 3243 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Leader pre-election won for term 2
09:09:45.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.375677 3243 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:09:45.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.375999 3243 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 1 FOLLOWER]: Advancing to term 2
09:09:45.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.382781 3243 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:45.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.384902 3188 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "30ebf93cb2f746f297dda1a92ead8278" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:45.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.385388 3188 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 1 FOLLOWER]: Advancing to term 2
09:09:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:45.387713 3071 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.388355 3243 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 election: Requested vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.391921 3188 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 30ebf93cb2f746f297dda1a92ead8278 in term 2.
09:09:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.392787 3069 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.393353 3243 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Leader election won for term 2
09:09:45.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.400997 3243 raft_consensus.cc:695] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 LEADER]: Becoming Leader. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Running, Role: LEADER
09:09:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.401666 3243 consensus_queue.cc:237] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:45.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.408659 3248 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 30ebf93cb2f746f297dda1a92ead8278. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:45.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.409682 3248 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: LEADER
09:09:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.411981 3250 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:09:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.229260 3237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:44.230888 3215 server_base.cc:1034] running on GCE node
09:09:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.423661 3250 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:09:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:44.230038 3239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.427378 3250 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.427608 3250 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:09:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.427549 3215 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:45.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.430327 3215 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:45.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.431372 3250 catalog_manager.cc:1505] Loading token signing keys...
09:09:45.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.431705 3215 hybrid_clock.cc:648] HybridClock initialized: now 1736845785431664 us; error 42 us; skew 500 ppm
09:09:45.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.432437 3215 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:45.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.434604 3250 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Loaded TSK: 0
09:09:45.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.435700 3250 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:09:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845785, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:45.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.441826 3215 init.cc:377] Logged in from keytab as kudu/127.2.167.254@KRBTEST.COM (short username kudu)
09:09:45.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.447248 3215 webserver.cc:458] Webserver started at http://127.2.167.254:33327/ using document root <none> and password file <none>
09:09:45.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.448206 3215 fs_manager.cc:362] Metadata directory not provided
09:09:45.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.448443 3215 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:45.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.455585 3215 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
09:09:45.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.459913 3256 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:45.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.460824 3215 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.461124 3215 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.463006 3215 fs_report.cc:389] FS layout report
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/data
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:45.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.518905 3215 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.524087 3215 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:45.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.524559 3215 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.615823 3215 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.254:33565
09:09:45.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.616003 3307 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.254:33565 every 8 connection(s)
09:09:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.618595 3215 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:45.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.628284 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3215
09:09:45.628 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:45.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.628911 3309 sys_catalog.cc:263] Verifying existing consensus state
09:09:45.630 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.167.253:39481(127.2.167.253:39481)] connecting to peer
09:09:45.631 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.167.252:37057(127.2.167.252:37057)] connecting to peer
09:09:45.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.634336 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap starting.
09:09:45.638 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.167.254:33565(127.2.167.254:33565)] connecting to peer
09:09:45.642 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.167.253:39481(127.2.167.253:39481)] Successfully connected to peer
09:09:45.656 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.167.252:37057(127.2.167.252:37057)] Successfully connected to peer
09:09:45.659 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.167.254:33565(127.2.167.254:33565)] Successfully connected to peer
09:09:45.678 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
09:09:45.679 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:45.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:45.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.685144 3309 log.cc:826] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Log is configured to *not* fsync() on all Append() calls
09:09:45.688 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:09:45.688 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
09:09:45.689 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
09:09:45.689 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.167.253
09:09:45.689 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:45.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:45.699 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:09:45.704 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
09:09:45.705 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.167.252
09:09:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:45.724 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
09:09:45.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.727376 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:45.727 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:45.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.728359 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap complete.
09:09:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:45.730 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:09:45.730 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
09:09:45.738 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.167.254
09:09:45.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.252@KRBTEST.COM
09:09:45.749 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xacf6e082, L:/127.0.0.1:52124 - R:/127.2.167.252:37057] using SASL/GSSAPI
09:09:45.751 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x703eed65, L:/127.0.0.1:47628 - R:/127.2.167.253:39481] using SASL/GSSAPI
09:09:45.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.763898 3309 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.767205 3309 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3130793f52ba4f65a4b1e245415a26b9, State: Initialized, Role: FOLLOWER
09:09:45.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
09:09:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.768203 3309 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:45.768 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=139), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.167.253:39481, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.167.252:37057, [9ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.167.254:33565, [124ms] Sub RPC ConnectToMaster: received response from server master-127.2.167.252:37057: OK, [137ms] Sub RPC ConnectToMaster: received response from server master-127.2.167.253:39481: OK, deferred=null) after lookup
09:09:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.771695 3317 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.772536 3317 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:45.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.773494 3309 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: configured and running, proceeding with master startup.
09:09:45.775 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdf52d1fd, L:/127.0.0.1:59920 - R:/127.2.167.254:33565] using SASL/GSSAPI
09:09:45.780 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.167.254:33565: Catalog manager is not initialized. State: Starting
09:09:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.799080 3328 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:45.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.799285 3328 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: loading cluster ID for follower catalog manager: success
09:09:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.804982 3328 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: acquiring CA information for follower catalog manager: success
09:09:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:45 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845785, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:45.810736 3328 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:46.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.012446 3283 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 1 FOLLOWER]: Advancing to term 2
09:09:46.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.014670 3188 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Refusing update from remote peer 30ebf93cb2f746f297dda1a92ead8278: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
09:09:46.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.016186 3331 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
09:09:46.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.020138 3283 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 2 FOLLOWER]: Refusing update from remote peer 30ebf93cb2f746f297dda1a92ead8278: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
09:09:46.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.021310 3331 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
09:09:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.030323 3317 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 30ebf93cb2f746f297dda1a92ead8278. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.031006 3317 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.040822 3333 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 30ebf93cb2f746f297dda1a92ead8278. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.041494 3333 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.044903 3335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.045670 3335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: LEADER
09:09:46.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.049104 3317 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.049903 3317 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:46.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.065593 3342 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.066546 3342 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: LEADER
09:09:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.069598 3333 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.070688 3333 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:46.774 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.253:39481
09:09:46.774 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:09:46.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.774917 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3048
09:09:46.800 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.167.253:39481(127.2.167.253:39481)] handling channelInactive
09:09:46.801 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.167.253:39481(127.2.167.253:39481)] cleaning up while in state READY due to: connection closed
09:09:46.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:46.801 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.252:37057
09:09:46.801 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:09:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.801903 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3122
09:09:46.833 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.167.252:37057(127.2.167.252:37057)] handling channelInactive
09:09:46.833 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.167.252:37057(127.2.167.252:37057)] cleaning up while in state READY due to: connection closed
09:09:46.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:46.834 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.254:33565
09:09:46.834 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:09:46.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.835204 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3215
09:09:46.853 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.167.254:33565(127.2.167.254:33565)] handling channelInactive
09:09:46.853 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.167.254:33565(127.2.167.254:33565)] cleaning up while in state READY due to: connection closed
09:09:46.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:46.853 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.253:39481
09:09:46.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:09:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:46.855132 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:46.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.139119 3343 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:47.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.139804 3343 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:47.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.140405 3343 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.168395 3343 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:47.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.168576 3343 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:47.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.168978 3343 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:47.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.169186 3343 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:47.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.169376 3343 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:47.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.169535 3343 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:47.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.169672 3343 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:47.202184 3343 master_runner.cc:386] Master server non-default flags:
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:47.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:47.203289 3343 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:47.204138 3343 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.217725 3350 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.219070 3352 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:48.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:47.217804 3349 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.303148 3351 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:09:48.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.303275 3343 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:48.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.306814 3343 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:48.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.309756 3343 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.311131 3343 hybrid_clock.cc:648] HybridClock initialized: now 1736845788311087 us; error 52 us; skew 500 ppm
09:09:48.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.311797 3343 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:48.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:48 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845788, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:48.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.324188 3343 init.cc:377] Logged in from keytab as kudu/127.2.167.253@KRBTEST.COM (short username kudu)
09:09:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.328325 3343 webserver.cc:458] Webserver started at http://127.2.167.253:39041/ using document root <none> and password file <none>
09:09:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.329069 3343 fs_manager.cc:362] Metadata directory not provided
09:09:48.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.329229 3343 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.335916 3343 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.001s sys 0.006s
09:09:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.339816 3361 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.340646 3343 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
09:09:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.340917 3343 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.342897 3343 fs_report.cc:389] FS layout report
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/data
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:48.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.391047 3343 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.395874 3343 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.396281 3343 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.461632 3343 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.253:39481
09:09:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.461707 3412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.253:39481 every 8 connection(s)
09:09:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.464090 3343 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:48.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.469132 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3343
09:09:48.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:48.469 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.252:37057
09:09:48.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.471406 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:48.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.476939 3413 sys_catalog.cc:263] Verifying existing consensus state
09:09:48.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.481637 3413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap starting.
09:09:48.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.513540 3413 log.cc:826] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Log is configured to *not* fsync() on all Append() calls
09:09:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.528009 3413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:48.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.528714 3413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap complete.
09:09:48.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.551098 3413 raft_consensus.cc:357] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:48.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.553581 3413 raft_consensus.cc:738] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Initialized, Role: FOLLOWER
09:09:48.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.554472 3413 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.557273 3417 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.558478 3417 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:48.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.558708 3413 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: configured and running, proceeding with master startup.
09:09:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.593874 3429 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:48.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.594494 3429 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: loading cluster ID for follower catalog manager: success
09:09:48.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.605052 3429 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: acquiring CA information for follower catalog manager: success
09:09:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.609907 3429 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.594084 3343 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:48.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.618175 3343 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.779913 3415 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.780416 3415 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.780998 3415 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.809113 3415 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.809310 3415 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.809592 3415 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.809788 3415 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:48.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.809962 3415 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:48.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.810132 3415 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:48.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.810261 3415 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.843258 3415 master_runner.cc:386] Master server non-default flags:
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:48.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.844405 3415 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.845203 3415 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:48.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.860718 3438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:50.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:48.862540 3415 server_base.cc:1034] running on GCE node
09:09:50.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.863056 3440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:50.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:48.861750 3437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:50.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.004727 3415 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.007908 3415 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.009352 3415 hybrid_clock.cc:648] HybridClock initialized: now 1736845790009315 us; error 30 us; skew 500 ppm
09:09:50.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.010259 3415 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:50 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845790, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:50.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.016639 3415 init.cc:377] Logged in from keytab as kudu/127.2.167.252@KRBTEST.COM (short username kudu)
09:09:50.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.021843 3415 webserver.cc:458] Webserver started at http://127.2.167.252:41465/ using document root <none> and password file <none>
09:09:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.023005 3415 fs_manager.cc:362] Metadata directory not provided
09:09:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.023236 3415 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.032209 3415 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
09:09:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.036899 3449 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:50.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.037945 3415 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.038297 3415 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.041091 3415 fs_report.cc:389] FS layout report
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/data
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:50.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.116446 3415 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:50.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.121618 3415 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.122035 3415 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:50.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.185024 3483 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.185667 3483 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:50.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.201452 3483 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:50.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.204892 3364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:50.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.213416 3362 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057): Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:50.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.214064 3364 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:50.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.214558 3364 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278; no voters: 3130793f52ba4f65a4b1e245415a26b9, 6356baec59b549689c7fa334e38f4a09
09:09:50.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.215587 3483 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
09:09:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.234470 3415 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.252:37057
09:09:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.234647 3504 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.252:37057 every 8 connection(s)
09:09:50.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.237108 3415 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:50.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.247419 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3415
09:09:50.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:50.248 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.254:33565
09:09:50.248 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:09:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.248934 3505 sys_catalog.cc:263] Verifying existing consensus state
09:09:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.249655 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:50.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:50.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:50.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.257181 3505 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap starting.
09:09:50.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.294473 3505 log.cc:826] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Log is configured to *not* fsync() on all Append() calls
09:09:50.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.308969 3505 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:50.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.309836 3505 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap complete.
09:09:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.331008 3505 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.333034 3505 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Initialized, Role: FOLLOWER
09:09:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.333811 3505 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.336529 3509 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.338148 3509 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.338349 3505 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: configured and running, proceeding with master startup.
09:09:50.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.373373 3520 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.373842 3520 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: loading cluster ID for follower catalog manager: success
09:09:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.381117 3520 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: acquiring CA information for follower catalog manager: success
09:09:50.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.385840 3520 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:50.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.370010 3415 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:50.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.393898 3415 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:50.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.595353 3507 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.595978 3507 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.596648 3507 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.625938 3507 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.626171 3507 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.626520 3507 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:50.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.626773 3507 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.627040 3507 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.627293 3507 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:50.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.627512 3507 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.662997 3507 master_runner.cc:386] Master server non-default flags:
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.664537 3507 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:50.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.665527 3507 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:50.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.681042 3529 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.489553 3536 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.490253 3536 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.494314 3536 leader_election.cc:290] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 30ebf93cb2f746f297dda1a92ead8278 (127.2.167.253:39481)
09:09:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:51.498452 3452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:51.517721 3452 leader_election.cc:336] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:51.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.525372 3388 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "30ebf93cb2f746f297dda1a92ead8278" is_pre_election: true
09:09:51.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.526486 3388 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6356baec59b549689c7fa334e38f4a09 in term 2.
09:09:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.528434 3452 leader_election.cc:304] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:51.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.529402 3536 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Leader pre-election won for term 3
09:09:51.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.529728 3536 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:09:51.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.530014 3536 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 2 FOLLOWER]: Advancing to term 3
09:09:51.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.538357 3536 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:51.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.539966 3536 leader_election.cc:290] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 election: Requested vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 30ebf93cb2f746f297dda1a92ead8278 (127.2.167.253:39481)
09:09:51.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.542106 3388 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6356baec59b549689c7fa334e38f4a09" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.542587 3388 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 2 FOLLOWER]: Advancing to term 3
09:09:51.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:51.548902 3452 leader_election.cc:336] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.550709 3388 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6356baec59b549689c7fa334e38f4a09 in term 3.
09:09:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.551553 3452 leader_election.cc:304] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.552387 3536 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Leader election won for term 3
09:09:51.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.554057 3536 raft_consensus.cc:695] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 LEADER]: Becoming Leader. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Running, Role: LEADER
09:09:51.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.554857 3536 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.578823 3541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.579550 3541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: LEADER
09:09:51.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.586246 3543 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:09:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.593840 3543 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:09:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.597571 3543 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:51.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.597808 3543 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:09:51.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.601855 3543 catalog_manager.cc:1505] Loading token signing keys...
09:09:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.604842 3543 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Loaded TSK: 0
09:09:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.605640 3543 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:09:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.681387 3528 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:50.682124 3531 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:50.683151 3507 server_base.cc:1034] running on GCE node
09:09:51.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.829097 3507 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:51.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.831979 3507 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.833360 3507 hybrid_clock.cc:648] HybridClock initialized: now 1736845791833329 us; error 35 us; skew 500 ppm
09:09:51.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.834028 3507 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:51 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845791, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.839745 3507 init.cc:377] Logged in from keytab as kudu/127.2.167.254@KRBTEST.COM (short username kudu)
09:09:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.845057 3507 webserver.cc:458] Webserver started at http://127.2.167.254:33327/ using document root <none> and password file <none>
09:09:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.845882 3507 fs_manager.cc:362] Metadata directory not provided
09:09:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.846076 3507 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.853441 3507 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
09:09:51.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.857740 3550 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.858886 3507 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.859166 3507 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.861560 3507 fs_report.cc:389] FS layout report
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/data
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.925910 3507 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:51.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.930951 3507 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:51.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.931288 3507 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.953938 3388 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Refusing update from remote peer 6356baec59b549689c7fa334e38f4a09: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
09:09:51.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:51.954113 3452 consensus_peers.cc:487] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 -> Peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Couldn't send request to peer 3130793f52ba4f65a4b1e245415a26b9. Status: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:09:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.955597 3541 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [LEADER]: Connected to new peer: Peer: permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
09:09:51.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.998067 3565 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:51.998849 3565 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.008061 3565 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.008746 3565 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.009850 3536 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.010444 3536 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: LEADER
09:09:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.057052 3507 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.254:33565
09:09:52.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.057142 3605 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.254:33565 every 8 connection(s)
09:09:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.060717 3507 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.062218 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3507
09:09:52.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:52.063 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.167.253:39481(127.2.167.253:39481)] connecting to peer
09:09:52.066 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.167.253:39481(127.2.167.253:39481)] Successfully connected to peer
09:09:52.070 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
09:09:52.070 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:52 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:52.074 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:09:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.073863 3606 sys_catalog.cc:263] Verifying existing consensus state
09:09:52.074 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
09:09:52.075 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.167.253
09:09:52.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.080055 3606 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap starting.
09:09:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:52 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845779, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.167.253@KRBTEST.COM
09:09:52.100 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x45db504d, L:/127.0.0.1:51330 - R:/127.2.167.253:39481] using SASL/GSSAPI
09:09:52.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.119838 3606 log.cc:826] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Log is configured to *not* fsync() on all Append() calls
09:09:52.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.133334 3606 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:52.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.133929 3606 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap complete.
09:09:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.157774 3606 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:52.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.160869 3606 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3130793f52ba4f65a4b1e245415a26b9, State: Initialized, Role: FOLLOWER
09:09:52.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.161823 3606 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.164425 3610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.165285 3610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:52.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.166069 3606 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: configured and running, proceeding with master startup.
09:09:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.190964 3622 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.191226 3622 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: loading cluster ID for follower catalog manager: success
09:09:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.198076 3622 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: acquiring CA information for follower catalog manager: success
09:09:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:52 dist-test-slave-bm4z krb5kdc[2730](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845791, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for kudu/127.2.167.254@KRBTEST.COM
09:09:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.202773 3622 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.503129 3581 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [term 2 FOLLOWER]: Advancing to term 3
09:09:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.517964 3630 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.519230 3630 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.522210 3610 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6356baec59b549689c7fa334e38f4a09. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.522776 3610 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9 [sys.catalog]: This master's current role is: FOLLOWER
09:09:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.945289 3541 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "6356baec59b549689c7fa334e38f4a09" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:52.946184 3541 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: LEADER
09:09:53.104 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.253:39481
09:09:53.104 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
09:09:53.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.105283 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3343
09:09:53.132 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.167.253:39481(127.2.167.253:39481)] handling channelInactive
09:09:53.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.167.253:39481(127.2.167.253:39481)] cleaning up while in state READY due to: connection closed
09:09:53.136 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:53.136 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.252:37057
09:09:53.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
09:09:53.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.137404 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3415
09:09:53.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:53.171 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.167.254:33565
09:09:53.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
09:09:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.171573 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3507
09:09:53.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:53.192 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.253:39481
09:09:53.192 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
09:09:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.193514 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:53.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:53.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:53.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.466192 3635 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:53.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.466781 3635 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:53.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.467633 3635 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.495922 3635 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.496179 3635 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.496531 3635 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:53.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.496804 3635 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.497054 3635 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.497301 3635 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.497504 3635 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.530956 3635 master_runner.cc:386] Master server non-default flags:
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.253.keytab
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.253
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.253:39481
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.253
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39041
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:53.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/logs
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:53.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.532490 3635 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:53.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:53.533457 3635 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:53.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.548619 3642 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.550025 3645 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:53.548689 3643 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:54.659065 3644 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
09:09:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.659173 3635 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.663476 3635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.665863 3635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.667177 3635 hybrid_clock.cc:648] HybridClock initialized: now 1736845794667130 us; error 55 us; skew 500 ppm
09:09:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.668234 3635 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:54 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845794, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:54.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.678406 3635 init.cc:377] Logged in from keytab as kudu/127.2.167.253@KRBTEST.COM (short username kudu)
09:09:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.683702 3635 webserver.cc:458] Webserver started at http://127.2.167.253:39041/ using document root <none> and password file <none>
09:09:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.684646 3635 fs_manager.cc:362] Metadata directory not provided
09:09:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.684862 3635 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:54.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.692122 3635 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.007s sys 0.000s
09:09:54.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.696142 3653 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.697042 3635 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.697324 3635 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30ebf93cb2f746f297dda1a92ead8278"
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:37 on dist-test-slave-bm4z"
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.699473 3635 fs_report.cc:389] FS layout report
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/wal
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/data
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.745090 3635 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:54.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.750401 3635 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:54.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.750794 3635 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.816833 3635 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.253:39481
09:09:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.816910 3704 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.253:39481 every 8 connection(s)
09:09:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.819518 3635 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-1/data/info.pb
09:09:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.827224 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3635
09:09:54.827 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:54.827 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.252:37057
09:09:54.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
09:09:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.828701 3705 sys_catalog.cc:263] Verifying existing consensus state
09:09:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.829161 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.834319 3705 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap starting.
09:09:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.879840 3705 log.cc:826] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Log is configured to *not* fsync() on all Append() calls
09:09:54.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.895403 3705 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:54.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.896258 3705 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Bootstrap complete.
09:09:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.917061 3705 raft_consensus.cc:357] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.919263 3705 raft_consensus.cc:738] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Initialized, Role: FOLLOWER
09:09:54.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.919864 3705 consensus_queue.cc:260] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.922719 3710 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.923570 3710 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: FOLLOWER
09:09:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.923815 3705 sys_catalog.cc:564] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: configured and running, proceeding with master startup.
09:09:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.968994 3721 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.969383 3721 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: loading cluster ID for follower catalog manager: success
09:09:54.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.946635 3635 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:54.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.970774 3635 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.976382 3721 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: acquiring CA information for follower catalog manager: success
09:09:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:54.981395 3721 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.142374 3707 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.142939 3707 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:55.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.143527 3707 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.188107 3707 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.188344 3707 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.188717 3707 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.188957 3707 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.189196 3707 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.189426 3707 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.189625 3707 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:55.223994 3707 master_runner.cc:386] Master server non-default flags:
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.252.keytab
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.252
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.252:37057
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:55.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.252
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41465
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/logs
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:55.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:55.225502 3707 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:55.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:55.226497 3707 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:55.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.240960 3730 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.122151 3735 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.122969 3735 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.140568 3735 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:56.145167 3656 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:56.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:56.174566 3656 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:56.181399 3654 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057): Network error: Client connection negotiation failed: client connection to 127.2.167.252:37057: connect: Connection refused (error 111)
09:09:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.181777 3654 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278; no voters: 3130793f52ba4f65a4b1e245415a26b9, 6356baec59b549689c7fa334e38f4a09
09:09:56.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.182621 3735 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
09:09:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.242046 3732 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:56.657121 3731 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1416 milliseconds
09:09:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.657218 3707 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:55.241317 3729 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.662678 3707 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.665530 3707 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.666945 3707 hybrid_clock.cc:648] HybridClock initialized: now 1736845796666882 us; error 68 us; skew 500 ppm
09:09:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.667809 3707 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:56.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:56 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845796, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.673358 3707 init.cc:377] Logged in from keytab as kudu/127.2.167.252@KRBTEST.COM (short username kudu)
09:09:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.678797 3707 webserver.cc:458] Webserver started at http://127.2.167.252:41465/ using document root <none> and password file <none>
09:09:56.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.679919 3707 fs_manager.cc:362] Metadata directory not provided
09:09:56.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.680202 3707 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:56.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.690264 3707 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
09:09:56.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.695191 3744 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.696226 3707 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
09:09:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.696480 3707 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:39 on dist-test-slave-bm4z"
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.698854 3707 fs_report.cc:389] FS layout report
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/wal
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/data
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.743304 3707 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:56.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.748349 3707 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:56.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.748756 3707 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.829417 3707 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.252:37057
09:09:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.829557 3797 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.252:37057 every 8 connection(s)
09:09:56.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.833380 3707 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-2/data/info.pb
09:09:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.843005 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3707
09:09:56.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:56.843 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.167.254:33565
09:09:56.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.844323 3798 sys_catalog.cc:263] Verifying existing consensus state
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.845332 2719 external_mini_cluster.cc:1351] Running /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:09:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.850816 3798 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap starting.
09:09:56.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.885142 3798 log.cc:826] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Log is configured to *not* fsync() on all Append() calls
09:09:56.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.902312 3798 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:09:56.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.902978 3798 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: Bootstrap complete.
09:09:56.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.925061 3798 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.927291 3798 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6356baec59b549689c7fa334e38f4a09, State: Initialized, Role: FOLLOWER
09:09:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.928371 3798 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:56.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.930976 3802 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.931950 3802 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.932765 3798 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: configured and running, proceeding with master startup.
09:09:56.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.971987 3813 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:56.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.972398 3813 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: loading cluster ID for follower catalog manager: success
09:09:56.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.987401 3813 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: acquiring CA information for follower catalog manager: success
09:09:57.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.996963 3813 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:09:57.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:56.975075 3707 master_runner.cc:186] Error getting master registration for 127.2.167.254:33565: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111),
09:09:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.017094 3707 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
09:09:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.299796 3800 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
09:09:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.300503 3800 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:09:57.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.301398 3800 flags.cc:425] Enabled unsafe flag: --never_fsync=true
09:09:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.353945 3800 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
09:09:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.354235 3800 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
09:09:57.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.354646 3800 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
09:09:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.354962 3800 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
09:09:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.355242 3800 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
09:09:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.355557 3800 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
09:09:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.355787 3800 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33975/jwks.json
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.415968 3800 master_runner.cc:386] Master server non-default flags:
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.167.212:33737
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.167.254:33565,127.2.167.253:39481,127.2.167.252:37057
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/kudu_127.2.167.254.keytab
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.167.254
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.167.254:33565
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:09:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33975/jwks.json
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.167.254
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33327
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/logs
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 622683201dc1b5c1b056d84565ee6cae8964ddd2
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Jan 2025 08:43:25 UTC on a633016ff5c9
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4172
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:09:57.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.417567 3800 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:09:57.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.418859 3800 file_cache.cc:492] Constructed file cache file cache with capacity 419430
09:09:57.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.449896 3822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.707073 3828 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:09:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.707933 3828 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.729243 3828 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:57.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.750936 3656 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:57.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.775903 3772 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "30ebf93cb2f746f297dda1a92ead8278" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "6356baec59b549689c7fa334e38f4a09" is_pre_election: true
09:09:57.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.777355 3772 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 30ebf93cb2f746f297dda1a92ead8278 in term 3.
09:09:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.779759 3654 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.780835 3828 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Leader pre-election won for term 4
09:09:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.781143 3828 raft_consensus.cc:491] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:09:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.781414 3828 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 3 FOLLOWER]: Advancing to term 4
09:09:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.790921 3828 raft_consensus.cc:513] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.794880 3772 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "30ebf93cb2f746f297dda1a92ead8278" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "6356baec59b549689c7fa334e38f4a09"
09:09:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.795666 3772 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 3 FOLLOWER]: Advancing to term 4
09:09:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.797658 3656 leader_election.cc:336] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111)
09:09:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.813663 3772 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 30ebf93cb2f746f297dda1a92ead8278 in term 4.
09:09:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.818296 3654 leader_election.cc:304] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 30ebf93cb2f746f297dda1a92ead8278, 6356baec59b549689c7fa334e38f4a09; no voters: 3130793f52ba4f65a4b1e245415a26b9
09:09:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.829135 3828 leader_election.cc:290] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [CANDIDATE]: Term 4 election: Requested vote from peers 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565), 6356baec59b549689c7fa334e38f4a09 (127.2.167.252:37057)
09:09:57.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.830618 3828 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 4 FOLLOWER]: Leader election won for term 4
09:09:57.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.833187 3828 raft_consensus.cc:695] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [term 4 LEADER]: Becoming Leader. State: Replica: 30ebf93cb2f746f297dda1a92ead8278, State: Running, Role: LEADER
09:09:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.834673 3828 consensus_queue.cc:237] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } }
09:09:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.852588 3833 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 30ebf93cb2f746f297dda1a92ead8278. Latest consensus state: current_term: 4 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.853596 3833 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: LEADER
09:09:57.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.858366 3835 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
09:09:57.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.873944 3835 catalog_manager.cc:1485] Initializing Kudu cluster ID...
09:09:57.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.882426 3835 catalog_manager.cc:1260] Loaded cluster ID: 53f7069cb6314a56839ba59e275e77b0
09:09:57.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.883675 3835 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
09:09:57.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.896502 3835 catalog_manager.cc:1505] Loading token signing keys...
09:09:57.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.903522 3835 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278: Loaded TSK: 0
09:09:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:57.906284 3835 catalog_manager.cc:1515] Initializing in-progress tserver states...
09:09:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.220605 3772 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [term 4 FOLLOWER]: Refusing update from remote peer 30ebf93cb2f746f297dda1a92ead8278: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
09:09:58.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.223320 3833 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
09:09:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.269745 3836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 30ebf93cb2f746f297dda1a92ead8278. Latest consensus state: current_term: 4 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:58.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.270591 3836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.271168 3828 sys_catalog.cc:455] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:58.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.272010 3828 sys_catalog.cc:458] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 [sys.catalog]: This master's current role is: LEADER
09:09:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:58.274641 3656 consensus_peers.cc:487] T 00000000000000000000000000000000 P 30ebf93cb2f746f297dda1a92ead8278 -> Peer 3130793f52ba4f65a4b1e245415a26b9 (127.2.167.254:33565): Couldn't send request to peer 3130793f52ba4f65a4b1e245415a26b9. Status: Network error: Client connection negotiation failed: client connection to 127.2.167.254:33565: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:09:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.289186 3836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "30ebf93cb2f746f297dda1a92ead8278" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3130793f52ba4f65a4b1e245415a26b9" member_type: VOTER last_known_addr { host: "127.2.167.254" port: 33565 } } peers { permanent_uuid: "30ebf93cb2f746f297dda1a92ead8278" member_type: VOTER last_known_addr { host: "127.2.167.253" port: 39481 } } peers { permanent_uuid: "6356baec59b549689c7fa334e38f4a09" member_type: VOTER last_known_addr { host: "127.2.167.252" port: 37057 } } }
09:09:58.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:58.290159 3836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6356baec59b549689c7fa334e38f4a09 [sys.catalog]: This master's current role is: FOLLOWER
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:58.840550 3821 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 3800
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.023644 3821 kernel_stack_watchdog.cc:198] Thread 3800 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 388ms:
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:09:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:57.451646 3823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.027799 3824 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1573 milliseconds
09:09:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.028281 3800 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.578s user 0.002s sys 0.000s
09:09:59.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.028659 3800 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.578s user 0.002s sys 0.000s
09:09:59.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.029991 3825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:09:59.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.030063 3800 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:09:59.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.031370 3800 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:09:59.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.036509 3800 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:09:59.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.037946 3800 hybrid_clock.cc:648] HybridClock initialized: now 1736845799037913 us; error 48 us; skew 500 ppm
09:09:59.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.038991 3800 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:09:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 14 09:09:59 dist-test-slave-bm4z krb5kdc[2730](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736845799, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.167.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:09:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.045887 3800 init.cc:377] Logged in from keytab as kudu/127.2.167.254@KRBTEST.COM (short username kudu)
09:09:59.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.051545 3800 webserver.cc:458] Webserver started at http://127.2.167.254:33327/ using document root <none> and password file <none>
09:09:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.052322 3800 fs_manager.cc:362] Metadata directory not provided
09:09:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.052487 3800 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:09:59.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.060232 3800 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.004s
09:09:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.064759 3849 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
09:09:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.065851 3800 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.066215 3800 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data,/tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3130793f52ba4f65a4b1e245415a26b9"
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-14 09:09:35 on dist-test-slave-bm4z"
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.069252 3800 fs_report.cc:389] FS layout report
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/wal
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/data
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:09:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:09:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:09:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:09:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:09:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:09:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.133270 3800 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
09:09:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.139642 3800 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:09:59.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.140148 3800 kserver.cc:163] Server-wide thread pool size limit: 3276
09:09:59.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.229650 3800 rpc_server.cc:307] RPC server started. Bound to: 127.2.167.254:33565
09:09:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.229852 3900 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.167.254:33565 every 8 connection(s)
09:09:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.232743 3800 server_base.cc:1166] Dumped server information to /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/master-0/data/info.pb
09:09:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.239465 2719 external_mini_cluster.cc:1413] Started /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu as pid 3800
09:09:59.240 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
09:09:59.240 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.167.253:39481(127.2.167.253:39481)] connecting to peer
09:09:59.242 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.167.253:39481(127.2.167.253:39481)] Successfully connected to peer
09:09:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.245452 3901 sys_catalog.cc:263] Verifying existing consensus state
09:09:59.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.252370 3901 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3130793f52ba4f65a4b1e245415a26b9: Bootstrap starting.
09:09:59.253 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
09:09:59.253 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4gMQ0A/test-tmp/mini-kudu-cluster2373831343960258170/krb5kdc/krb5cc
09:09:59.254 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
09:09:59.254 [WARN - kudu-nio-0] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
09:09:59.255 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.167.253:39481(127.2.167.253:39481)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:09:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250114 09:09:59.258195 3903 negotiation.cc:337] Failed RPC negotiation. Trace:
09:09:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.242195 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:60378 (local address 127.2.167.253:39481)
09:09:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.246947 (+ 4752us) server_negotiation.cc:206] Beginning negotiation
09:09:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.246977 (+ 30us) server_negotiation.cc:399] Waiting for connection header
09:09:59.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.247030 (+ 53us) server_negotiation.cc:407] Connection header received
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.247188 (+ 158us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.247216 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.247562 (+ 346us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.256182 (+ 8620us) server_negotiation.cc:226] Negotiated authn=SASL
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0114 09:09:59.256852 (+ 670us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:60378: BlockingRecv error: recv got EOF from 127.0.0.1:60378 (error 108)
09:09:59.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4125,"thread_start_us":3818,"threads_started":1}
09:09:59.261 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.167.253:39481(127.2.167.253:39481)] handling channelInactive
09:09:59.261 [DEBUG - main] (Connection.java:575) [peer master-127.2.167.253:39481(127.2.167.253:39481)] disconnecting while in state TERMINATED
09:09:59.261 [DEBUG - main] (Connection.java:575) [peer master-127.2.167.254:33565(127.2.167.254:33565)] disconnecting while in state TERMINATED
09:09:59.261 [DEBUG - main] (Connection.java:575) [peer master-127.2.167.252:37057(127.2.167.252:37057)] disconnecting while in state TERMINATED
09:09:59.263 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:09:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.274782 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3800
09:09:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.291299 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3635
09:09:59.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:09:59.321028 2719 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 3707
09:10:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:10:00Z chronyd exiting
09:10:01.450 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:10:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.451846 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2272
09:10:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.475924 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2427
09:10:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.498046 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2573
09:10:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.519754 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2003
09:10:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.545028 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2074
09:10:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250114 09:10:01.570653 1991 external_mini_cluster.cc:1620] Killing /tmp/dist-test-task4gMQ0A/build/tsan/bin/kudu with pid 2149
09:10:02.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-14T09:10:02Z chronyd exiting
09:10:03.641 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more