15:11:48.420 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:11:48.420 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskZW0gWW/test-tmp
15:11:48.420 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:11:48.421 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskZW0gWW/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:11:48.422 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:11:48.732 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:11:48.732 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:11:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:48.735244 3493 mini_oidc.cc:150] Starting JWKS server
15:11:48.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:48.738502 3493 webserver.cc:458] Webserver started at http://127.0.0.1:44389/ using document root <none> and password file <none>
15:11:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:48.738936 3493 mini_oidc.cc:186] Starting OIDC Discovery server
15:11:48.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:48.740639 3493 webserver.cc:458] Webserver started at http://127.0.0.1:39751/ using document root <none> and password file <none>
15:11:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:11:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:11:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:11:48Z Disabled control of system clock
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:48.799438 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:48.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/logs
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data/info.pb
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.126:35551
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.126
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json with env {}
15:11:49.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.091531 3505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.092170 3505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.092764 3505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.123860 3505 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.124224 3505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.124547 3505 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.124817 3505 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.125092 3505 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.125305 3505 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44389/jwks.json
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:49.159897 3505 master_runner.cc:386] Master server non-default flags:
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.126:35551
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data/info.pb
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.126
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:49.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/logs
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:49.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:49.161458 3505 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:49.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:49.162997 3505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:49.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.177558 3511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:50.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.177579 3512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:50.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:49.179029 3505 server_base.cc:1034] running on GCE node
15:11:50.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:49.178639 3514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.267925 3505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:50.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.270865 3505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.272286 3505 hybrid_clock.cc:648] HybridClock initialized: now 1736694710272235 us; error 63 us; skew 500 ppm
15:11:50.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.273056 3505 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.283198 3505 webserver.cc:458] Webserver started at http://127.3.105.126:34423/ using document root <none> and password file <none>
15:11:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.284169 3505 fs_manager.cc:362] Metadata directory not provided
15:11:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.284373 3505 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:50.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.284772 3505 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.289661 3505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data/instance:
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "104f24a8c7a048ddbce478730ed15e02"
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:50 on dist-test-slave-xd94"
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.290724 3505 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal/instance:
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "104f24a8c7a048ddbce478730ed15e02"
15:11:50.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:50 on dist-test-slave-xd94"
15:11:50.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.297731 3505 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
15:11:50.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.303117 3521 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.304256 3505 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:11:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.304550 3505 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "104f24a8c7a048ddbce478730ed15e02"
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:50 on dist-test-slave-xd94"
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.304850 3505 fs_report.cc:389] FS layout report
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data/data
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:50.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.366467 3505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.367918 3505 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.368321 3505 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:50.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.434346 3505 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.126:35551
15:11:50.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.434412 3573 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.126:35551 every 8 connection(s)
15:11:50.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.437050 3505 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/data/info.pb
15:11:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.441988 3574 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:11:50.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.446219 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 3505
15:11:50.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.446756 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-0/wal/instance
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.451967 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/logs
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data/info.pb
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.125:34619
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.125
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:50.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:50.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json with env {}
15:11:50.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.456563 3574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } has no permanent_uuid. Determining permanent_uuid...
15:11:50.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.480440 3574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } has no permanent_uuid. Determining permanent_uuid...
15:11:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.482529 3522 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:50.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.486425 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.535979 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 1
15:11:50.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.539534 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:50.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.607985 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 2
15:11:50.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.611750 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:50.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.703248 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 3
15:11:50.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.707001 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.750479 3576 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.751026 3576 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:50.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.751554 3576 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.782300 3576 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.782594 3576 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.782799 3576 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.782989 3576 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.783176 3576 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.783317 3576 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44389/jwks.json
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.818693 3576 master_runner.cc:386] Master server non-default flags:
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.125:34619
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data/info.pb
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.125
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/logs
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.820031 3576 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:50.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.821514 3576 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:50.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.836841 3588 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:50.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.850483 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 4
15:11:50.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.854292 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.153810 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 5
15:11:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:51.157868 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.705418 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 6
15:11:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:51.714018 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.125:34619: Network error: Client connection negotiation failed: client connection to 127.3.105.125:34619: connect: Connection refused (error 111)
15:11:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.837355 3586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:50.837992 3576 server_base.cc:1034] running on GCE node
15:11:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:50.837626 3585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:51.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.930344 3576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:51.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.933101 3576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.934446 3576 hybrid_clock.cc:648] HybridClock initialized: now 1736694711934407 us; error 45 us; skew 500 ppm
15:11:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.935213 3576 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.941686 3576 webserver.cc:458] Webserver started at http://127.3.105.125:42917/ using document root <none> and password file <none>
15:11:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.942624 3576 fs_manager.cc:362] Metadata directory not provided
15:11:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.942822 3576 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.943207 3576 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:11:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.948153 3576 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data/instance:
15:11:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf"
15:11:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:51 on dist-test-slave-xd94"
15:11:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.949225 3576 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal/instance:
15:11:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf"
15:11:51.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:51 on dist-test-slave-xd94"
15:11:51.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.956338 3576 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
15:11:51.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.961726 3597 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.962762 3576 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.963124 3576 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf"
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:51 on dist-test-slave-xd94"
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:51.963501 3576 fs_report.cc:389] FS layout report
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal
15:11:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data/data
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.013548 3576 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.015007 3576 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.015481 3576 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:52.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.082993 3576 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.125:34619
15:11:52.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.083057 3648 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.125:34619 every 8 connection(s)
15:11:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.085774 3576 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/data/info.pb
15:11:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.090759 3649 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:11:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.095659 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 3576
15:11:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.095988 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-1/wal/instance
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.097978 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/logs
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data/info.pb
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.124:46531
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.124
15:11:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json with env {}
15:11:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.102355 3649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } has no permanent_uuid. Determining permanent_uuid...
15:11:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.124369 3649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } has no permanent_uuid. Determining permanent_uuid...
15:11:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.138028 3649 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } has no permanent_uuid. Determining permanent_uuid...
15:11:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.140560 3601 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.144826 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.194288 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 1
15:11:52.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.198344 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.266752 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 2
15:11:52.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.270782 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.362183 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 3
15:11:52.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.365998 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.402330 3651 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.402940 3651 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.403548 3651 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:52.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.434358 3651 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:52.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.434697 3651 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.434986 3651 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.435322 3651 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.435782 3651 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:52.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.436075 3651 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44389/jwks.json
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.471225 3651 master_runner.cc:386] Master server non-default flags:
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:52.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.124:46531
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44389/jwks.json
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data/info.pb
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.124
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/logs
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:52.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.472790 3651 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:52.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.474335 3651 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:52.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.488890 3664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:52.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.509338 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 4
15:11:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.513144 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.774586 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } attempt: 7
15:11:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.789016 3574 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } has no permanent_uuid. Determining permanent_uuid...
15:11:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.795137 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.812635 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 5
15:11:52.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.816702 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.853569 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 1
15:11:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.857702 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.939061 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 2
15:11:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.943126 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.028517 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 3
15:11:53.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.032732 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:53.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.173116 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 4
15:11:53.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.177224 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.364125 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 6
15:11:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.370321 3649 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:53.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.460603 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 5
15:11:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.464311 3574 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.105.124:46531: Network error: Client connection negotiation failed: client connection to 127.3.105.124:46531: connect: Connection refused (error 111)
15:11:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.488960 3662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:52.489326 3661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:53.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:52.489796 3651 server_base.cc:1034] running on GCE node
15:11:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.594274 3651 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:53.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.596959 3651 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.598335 3651 hybrid_clock.cc:648] HybridClock initialized: now 1736694713598281 us; error 54 us; skew 500 ppm
15:11:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.599161 3651 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.605684 3651 webserver.cc:458] Webserver started at http://127.3.105.124:36795/ using document root <none> and password file <none>
15:11:53.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.606688 3651 fs_manager.cc:362] Metadata directory not provided
15:11:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.606906 3651 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.607412 3651 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:11:53.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.612221 3651 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data/instance:
15:11:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c218c8a068a444738a4bd09adbf30c0c"
15:11:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:53 on dist-test-slave-xd94"
15:11:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.613494 3651 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal/instance:
15:11:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c218c8a068a444738a4bd09adbf30c0c"
15:11:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:53 on dist-test-slave-xd94"
15:11:53.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.620669 3651 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
15:11:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.626161 3677 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:53.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.627230 3651 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
15:11:53.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.627605 3651 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c218c8a068a444738a4bd09adbf30c0c"
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:53 on dist-test-slave-xd94"
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.627933 3651 fs_report.cc:389] FS layout report
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data/data
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:53.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.680193 3651 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.681672 3651 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:53.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.682122 3651 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.748409 3651 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.124:46531
15:11:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.748514 3728 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.124:46531 every 8 connection(s)
15:11:53.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.751129 3651 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/data/info.pb
15:11:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.756119 3729 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:11:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.758800 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 3651
15:11:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.759183 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/master-2/wal/instance
15:11:53.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.769894 3729 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } has no permanent_uuid. Determining permanent_uuid...
15:11:53.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.799940 3729 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } has no permanent_uuid. Determining permanent_uuid...
15:11:53.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.808967 3729 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } has no permanent_uuid. Determining permanent_uuid...
15:11:53.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.824524 3729 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: Bootstrap starting.
15:11:53.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.830842 3729 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: Neither blocks nor log segments found. Creating new log.
15:11:53.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.832589 3729 log.cc:826] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: Log is configured to *not* fsync() on all Append() calls
15:11:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.836858 3729 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: No bootstrap required, opened a new log
15:11:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.858223 3729 raft_consensus.cc:357] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.859108 3729 raft_consensus.cc:383] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:11:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.859411 3729 raft_consensus.cc:738] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c218c8a068a444738a4bd09adbf30c0c, State: Initialized, Role: FOLLOWER
15:11:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.860246 3729 consensus_queue.cc:260] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.862531 3738 sys_catalog.cc:455] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:53.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.863152 3738 sys_catalog.cc:458] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: This master's current role is: FOLLOWER
15:11:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.863947 3729 sys_catalog.cc:564] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: configured and running, proceeding with master startup.
15:11:53.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.889634 3749 catalog_manager.cc:1559] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:11:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.889909 3749 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:11:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.885486 3651 master_runner.cc:186] Error getting master registration for 127.3.105.126:35551: OK, instance_id { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" instance_seqno: 1736694710416096 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:11:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:53.917804 3651 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:53.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:53.936987 3683 tablet.cc:2367] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:11:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.016682 3574 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 6
15:11:54.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.031876 3574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: Bootstrap starting.
15:11:54.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.039149 3574 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: Neither blocks nor log segments found. Creating new log.
15:11:54.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.040951 3574 log.cc:826] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: Log is configured to *not* fsync() on all Append() calls
15:11:54.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.045461 3574 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: No bootstrap required, opened a new log
15:11:54.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.066408 3574 raft_consensus.cc:357] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.067065 3574 raft_consensus.cc:383] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:11:54.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.067401 3574 raft_consensus.cc:738] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 104f24a8c7a048ddbce478730ed15e02, State: Initialized, Role: FOLLOWER
15:11:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.068217 3574 consensus_queue.cc:260] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.071081 3753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.071969 3753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.072904 3574 sys_catalog.cc:564] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: configured and running, proceeding with master startup.
15:11:54.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.105855 3764 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:11:54.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.106381 3764 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:11:54.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.116158 3738 raft_consensus.cc:491] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:54.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.116717 3738 raft_consensus.cc:513] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.121062 3549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c218c8a068a444738a4bd09adbf30c0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "104f24a8c7a048ddbce478730ed15e02" is_pre_election: true
15:11:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.122009 3549 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c218c8a068a444738a4bd09adbf30c0c in term 0.
15:11:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.123667 3681 leader_election.cc:304] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 104f24a8c7a048ddbce478730ed15e02, c218c8a068a444738a4bd09adbf30c0c; no voters:
15:11:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.126179 3738 leader_election.cc:290] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 104f24a8c7a048ddbce478730ed15e02 (127.3.105.126:35551), 20a46f292dcb43f1bb8a8c9d4e63e7cf (127.3.105.125:34619)
15:11:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.126542 3527 tablet.cc:2367] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:11:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.126813 3738 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Leader pre-election won for term 1
15:11:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.127136 3738 raft_consensus.cc:491] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:11:54.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.127456 3738 raft_consensus.cc:3054] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 0 FOLLOWER]: Advancing to term 1
15:11:54.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.134420 3738 raft_consensus.cc:513] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.136821 3738 leader_election.cc:290] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [CANDIDATE]: Term 1 election: Requested vote from peers 104f24a8c7a048ddbce478730ed15e02 (127.3.105.126:35551), 20a46f292dcb43f1bb8a8c9d4e63e7cf (127.3.105.125:34619)
15:11:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.140226 3549 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c218c8a068a444738a4bd09adbf30c0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "104f24a8c7a048ddbce478730ed15e02"
15:11:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.140807 3549 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 0 FOLLOWER]: Advancing to term 1
15:11:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.147392 3549 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c218c8a068a444738a4bd09adbf30c0c in term 1.
15:11:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.148623 3681 leader_election.cc:304] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 104f24a8c7a048ddbce478730ed15e02, c218c8a068a444738a4bd09adbf30c0c; no voters:
15:11:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.149576 3738 raft_consensus.cc:2798] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 1 FOLLOWER]: Leader election won for term 1
15:11:54.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.150123 3738 raft_consensus.cc:695] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [term 1 LEADER]: Becoming Leader. State: Replica: c218c8a068a444738a4bd09adbf30c0c, State: Running, Role: LEADER
15:11:54.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.151765 3738 consensus_queue.cc:237] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.165570 3766 sys_catalog.cc:455] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: SysCatalogTable state changed. Reason: New leader c218c8a068a444738a4bd09adbf30c0c. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.166735 3766 sys_catalog.cc:458] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: This master's current role is: LEADER
15:11:54.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.174432 3768 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:11:54.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.182327 3768 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:11:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.210542 3549 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [term 1 FOLLOWER]: Refusing update from remote peer c218c8a068a444738a4bd09adbf30c0c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:11:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.104292 3505 master_runner.cc:186] Error getting master registration for 127.3.105.125:34619: OK, instance_id { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" instance_seqno: 1736694712064294 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:11:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.212560 3766 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:11:54.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.213613 3505 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.255240 3769 mvcc.cc:204] Tried to move back new op lower bound from 7113501549368324096 to 7113501549178175488. Current Snapshot: MvccSnapshot[applied={T|T < 7113501549368324096}]
15:11:54.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.260809 3753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c218c8a068a444738a4bd09adbf30c0c. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.261615 3753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.269757 3738 sys_catalog.cc:455] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.270540 3738 sys_catalog.cc:458] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: This master's current role is: LEADER
15:11:54.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.279350 3768 catalog_manager.cc:1348] Generated new cluster ID: f9e10461b8ae4f1c97e6efa883dde69b
15:11:54.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.279729 3768 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:11:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.301246 3753 sys_catalog.cc:455] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.302033 3753 sys_catalog.cc:458] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02 [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.329238 3768 catalog_manager.cc:1371] Generated new certificate authority record
15:11:54.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.333410 3768 catalog_manager.cc:1505] Loading token signing keys...
15:11:54.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.366385 3768 catalog_manager.cc:5899] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c: Generated new TSK 0
15:11:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.369125 3768 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.402628 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/logs
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data/info.pb
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.65:0
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.65
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.65
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:54.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:11:54.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.430832 3649 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } attempt: 7
15:11:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.483281 3649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: Bootstrap starting.
15:11:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.501639 3649 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: Neither blocks nor log segments found. Creating new log.
15:11:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.507207 3649 log.cc:826] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: Log is configured to *not* fsync() on all Append() calls
15:11:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.530390 3649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: No bootstrap required, opened a new log
15:11:54.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.611763 3649 raft_consensus.cc:357] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.614238 3649 raft_consensus.cc:383] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:11:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.615125 3649 raft_consensus.cc:738] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 20a46f292dcb43f1bb8a8c9d4e63e7cf, State: Initialized, Role: FOLLOWER
15:11:54.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.618394 3649 consensus_queue.cc:260] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } }
15:11:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.632403 3778 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.633231 3778 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.639887 3624 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c218c8a068a444738a4bd09adbf30c0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" is_pre_election: true
15:11:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.640805 3624 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c218c8a068a444738a4bd09adbf30c0c in term 0.
15:11:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.641753 3623 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c218c8a068a444738a4bd09adbf30c0c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf"
15:11:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.642324 3623 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 0 FOLLOWER]: Advancing to term 1
15:11:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.650321 3623 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c218c8a068a444738a4bd09adbf30c0c in term 1.
15:11:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.651595 3622 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [term 1 FOLLOWER]: Refusing update from remote peer c218c8a068a444738a4bd09adbf30c0c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:11:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.657945 3766 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [LEADER]: Connected to new peer: Peer: permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:11:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.659186 3649 sys_catalog.cc:564] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: configured and running, proceeding with master startup.
15:11:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.716909 3778 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: SysCatalogTable state changed. Reason: New leader c218c8a068a444738a4bd09adbf30c0c. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.717691 3778 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.717799 3783 sys_catalog.cc:455] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.718608 3783 sys_catalog.cc:458] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf [sys.catalog]: This master's current role is: FOLLOWER
15:11:54.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.831992 3794 catalog_manager.cc:1260] Loaded cluster ID: f9e10461b8ae4f1c97e6efa883dde69b
15:11:54.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.832273 3794 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: loading cluster ID for follower catalog manager: success
15:11:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.847947 3794 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: acquiring CA information for follower catalog manager: success
15:11:54.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:54.867697 3794 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 20a46f292dcb43f1bb8a8c9d4e63e7cf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:54.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.887543 3775 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:54.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.888475 3775 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:54.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.889430 3775 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:54.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.953369 3775 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:54.954979 3775 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.105.65
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.022080 3775 tablet_server_runner.cc:78] Tablet server non-default flags:
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:55.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.65:0
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data/info.pb
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.65
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.65
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/logs
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.026453 3775 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:55.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.028990 3775 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:55.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:55.077659 3801 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.118702 3764 catalog_manager.cc:1260] Loaded cluster ID: f9e10461b8ae4f1c97e6efa883dde69b
15:11:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.119261 3764 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: loading cluster ID for follower catalog manager: success
15:11:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.127259 3764 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: acquiring CA information for follower catalog manager: success
15:11:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.141731 3764 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 104f24a8c7a048ddbce478730ed15e02: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:55.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.269346 3766 sys_catalog.cc:455] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c218c8a068a444738a4bd09adbf30c0c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "104f24a8c7a048ddbce478730ed15e02" member_type: VOTER last_known_addr { host: "127.3.105.126" port: 35551 } } peers { permanent_uuid: "20a46f292dcb43f1bb8a8c9d4e63e7cf" member_type: VOTER last_known_addr { host: "127.3.105.125" port: 34619 } } peers { permanent_uuid: "c218c8a068a444738a4bd09adbf30c0c" member_type: VOTER last_known_addr { host: "127.3.105.124" port: 46531 } } }
15:11:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:55.270555 3766 sys_catalog.cc:458] T 00000000000000000000000000000000 P c218c8a068a444738a4bd09adbf30c0c [sys.catalog]: This master's current role is: LEADER
15:11:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:55.084107 3802 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:56.451565 3800 debug-util.cc:398] Leaking SignalData structure 0x7b0800006bc0 after lost signal to thread 3775
15:11:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:56.941466 3775 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 1.873s user 0.567s sys 0.813s
15:11:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:56.942389 3775 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 1.874s user 0.568s sys 0.813s
15:11:56.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:56.954638 3810 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:56.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.955937 3775 server_base.cc:1034] running on GCE node
15:11:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.957267 3775 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.959868 3775 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:56.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.961390 3775 hybrid_clock.cc:648] HybridClock initialized: now 1736694716961282 us; error 106 us; skew 500 ppm
15:11:56.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.962536 3775 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:56.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.974227 3775 webserver.cc:458] Webserver started at http://127.3.105.65:34741/ using document root <none> and password file <none>
15:11:56.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.975401 3775 fs_manager.cc:362] Metadata directory not provided
15:11:56.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.975675 3775 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.976179 3775 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:11:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.983042 3775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data/instance:
15:11:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53bdfc1797684486b965f72995e6ff28"
15:11:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:56 on dist-test-slave-xd94"
15:11:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.984556 3775 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal/instance:
15:11:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53bdfc1797684486b965f72995e6ff28"
15:11:56.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:56 on dist-test-slave-xd94"
15:11:56.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:56.994385 3775 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.003s sys 0.007s
15:11:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.001819 3816 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:57.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.003134 3775 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.003559 3775 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53bdfc1797684486b965f72995e6ff28"
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:56 on dist-test-slave-xd94"
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.004004 3775 fs_report.cc:389] FS layout report
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data/data
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:57.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:57.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.078696 3775 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.080730 3775 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.081279 3775 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.090376 3775 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:11:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.108554 3775 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:11:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.108851 3775 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:11:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.109154 3775 ts_tablet_manager.cc:610] Registered 0 tablets
15:11:57.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.109369 3775 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:11:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.283071 3775 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.65:36005
15:11:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.283133 3928 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.65:36005 every 8 connection(s)
15:11:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.286589 3775 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/data/info.pb
15:11:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.295702 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 3775
15:11:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.296178 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-0/wal/instance
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.315752 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:57.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/logs
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data/info.pb
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.66:0
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.66
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.66
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:57.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:11:57.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.362005 3930 heartbeater.cc:346] Connected to a master server at 127.3.105.125:34619
15:11:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.362509 3930 heartbeater.cc:463] Registering TS with master...
15:11:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.363831 3930 heartbeater.cc:510] Master 127.3.105.125:34619 requested a full tablet report, sending...
15:11:57.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.367648 3929 heartbeater.cc:346] Connected to a master server at 127.3.105.124:46531
15:11:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.367997 3929 heartbeater.cc:463] Registering TS with master...
15:11:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.367940 3614 ts_manager.cc:194] Registered new tserver with Master: 53bdfc1797684486b965f72995e6ff28 (127.3.105.65:36005)
15:11:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.368913 3929 heartbeater.cc:510] Master 127.3.105.124:46531 requested a full tablet report, sending...
15:11:57.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.371430 3694 ts_manager.cc:194] Registered new tserver with Master: 53bdfc1797684486b965f72995e6ff28 (127.3.105.65:36005)
15:11:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.374938 3694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.105.65:36421
15:11:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.375836 3931 heartbeater.cc:346] Connected to a master server at 127.3.105.126:35551
15:11:57.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.376197 3931 heartbeater.cc:463] Registering TS with master...
15:11:57.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.377096 3931 heartbeater.cc:510] Master 127.3.105.126:35551 requested a full tablet report, sending...
15:11:57.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.380051 3539 ts_manager.cc:194] Registered new tserver with Master: 53bdfc1797684486b965f72995e6ff28 (127.3.105.65:36005)
15:11:57.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.657253 3936 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:57.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.657773 3936 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:57.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.658300 3936 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.690634 3936 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.691589 3936 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.105.66
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.727303 3936 tablet_server_runner.cc:78] Tablet server non-default flags:
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal
15:11:57.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.66:0
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data/info.pb
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.66
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:57.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.66
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/logs
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:57.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.728979 3936 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:57.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.730726 3936 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:57.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.750147 3947 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:58.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.378831 3929 heartbeater.cc:502] Master 127.3.105.124:46531 was elected leader, sending a full tablet report...
15:11:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:57.759279 3936 server_base.cc:1034] running on GCE node
15:11:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.758059 3946 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:58.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:57.763696 3949 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.849409 3936 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:58.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.851992 3936 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.853417 3936 hybrid_clock.cc:648] HybridClock initialized: now 1736694718853373 us; error 57 us; skew 500 ppm
15:11:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.854184 3936 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:58.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.860591 3936 webserver.cc:458] Webserver started at http://127.3.105.66:44247/ using document root <none> and password file <none>
15:11:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.861528 3936 fs_manager.cc:362] Metadata directory not provided
15:11:58.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.861725 3936 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:58.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.862108 3936 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:11:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.867096 3936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data/instance:
15:11:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d7301174324fb18690d16f9e361a2b"
15:11:58.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:58 on dist-test-slave-xd94"
15:11:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.868239 3936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal/instance:
15:11:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d7301174324fb18690d16f9e361a2b"
15:11:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:58 on dist-test-slave-xd94"
15:11:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.875694 3936 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:11:58.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.881196 3957 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.882259 3936 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:11:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.882555 3936 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal
15:11:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5d7301174324fb18690d16f9e361a2b"
15:11:58.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:11:58 on dist-test-slave-xd94"
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.882861 3936 fs_report.cc:389] FS layout report
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data/data
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:58.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:58.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.937242 3936 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.938657 3936 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.939052 3936 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:58.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.941445 3936 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:11:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.945538 3936 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:11:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.945725 3936 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:11:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.945930 3936 ts_tablet_manager.cc:610] Registered 0 tablets
15:11:58.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:58.946115 3936 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:11:59.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.076925 3936 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.66:41419
15:11:59.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.077059 4070 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.66:41419 every 8 connection(s)
15:11:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.079988 3936 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/data/info.pb
15:11:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.085878 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 3936
15:11:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.086521 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-1/wal/instance
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.108906 3493 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/logs
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data/info.pb
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.67:0
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.67
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.67
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:59.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:11:59.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.144897 4074 heartbeater.cc:346] Connected to a master server at 127.3.105.126:35551
15:11:59.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.145434 4074 heartbeater.cc:463] Registering TS with master...
15:11:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.146759 4074 heartbeater.cc:510] Master 127.3.105.126:35551 requested a full tablet report, sending...
15:11:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.150213 3539 ts_manager.cc:194] Registered new tserver with Master: b5d7301174324fb18690d16f9e361a2b (127.3.105.66:41419)
15:11:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.152370 4073 heartbeater.cc:346] Connected to a master server at 127.3.105.125:34619
15:11:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.152717 4073 heartbeater.cc:463] Registering TS with master...
15:11:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.153527 4071 heartbeater.cc:346] Connected to a master server at 127.3.105.124:46531
15:11:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.153599 4073 heartbeater.cc:510] Master 127.3.105.125:34619 requested a full tablet report, sending...
15:11:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.153905 4071 heartbeater.cc:463] Registering TS with master...
15:11:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.154762 4071 heartbeater.cc:510] Master 127.3.105.124:46531 requested a full tablet report, sending...
15:11:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.155936 3614 ts_manager.cc:194] Registered new tserver with Master: b5d7301174324fb18690d16f9e361a2b (127.3.105.66:41419)
15:11:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.156876 3694 ts_manager.cc:194] Registered new tserver with Master: b5d7301174324fb18690d16f9e361a2b (127.3.105.66:41419)
15:11:59.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.158598 3694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.105.66:57333
15:11:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.442838 4080 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:59.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.443394 4080 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:59.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.443931 4080 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.475793 4080 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:59.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.476696 4080 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.105.67
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.511026 4080 tablet_server_runner.cc:78] Tablet server non-default flags:
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.105.84:39585
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.105.67:0
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data/info.pb
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.105.67
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.105.126:35551,127.3.105.125:34619,127.3.105.124:46531
15:11:59.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.105.67
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/logs
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.512668 4080 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:11:59.514251 4080 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.530140 4088 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.161568 4071 heartbeater.cc:502] Master 127.3.105.124:46531 was elected leader, sending a full tablet report...
15:12:00.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.530467 4089 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:00.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:11:59.532006 4091 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:00.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:00.615758 4090 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:12:00.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.615869 4080 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:12:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.619205 4080 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.621384 4080 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:00.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.622740 4080 hybrid_clock.cc:648] HybridClock initialized: now 1736694720622697 us; error 42 us; skew 500 ppm
15:12:00.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.623591 4080 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:00.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.629711 4080 webserver.cc:458] Webserver started at http://127.3.105.67:39997/ using document root <none> and password file <none>
15:12:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.630648 4080 fs_manager.cc:362] Metadata directory not provided
15:12:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.630858 4080 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.631400 4080 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:12:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.636802 4080 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data/instance:
15:12:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d74e98e0e83b41619390a0b8f78b6edc"
15:12:00.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:00 on dist-test-slave-xd94"
15:12:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.637858 4080 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal/instance:
15:12:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d74e98e0e83b41619390a0b8f78b6edc"
15:12:00.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:00 on dist-test-slave-xd94"
15:12:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.644596 4080 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
15:12:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.649801 4100 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.650786 4080 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.651069 4080 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d74e98e0e83b41619390a0b8f78b6edc"
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:00 on dist-test-slave-xd94"
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.651405 4080 fs_report.cc:389] FS layout report
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data/data
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:00.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.698758 4080 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:00.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.700161 4080 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:00.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.700568 4080 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:00.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.702885 4080 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:12:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.706833 4080 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:12:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.707044 4080 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:12:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.707248 4080 ts_tablet_manager.cc:610] Registered 0 tablets
15:12:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.707410 4080 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:12:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.843583 4080 rpc_server.cc:307] RPC server started. Bound to: 127.3.105.67:34201
15:12:00.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.843691 4213 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.3.105.67:34201 every 8 connection(s)
15:12:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.845885 4080 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/data/info.pb
15:12:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.846374 3493 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4080
15:12:00.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.846683 3493 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster8049003332815559525/ts-2/wal/instance
15:12:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.887811 4214 heartbeater.cc:346] Connected to a master server at 127.3.105.124:46531
15:12:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.888268 4214 heartbeater.cc:463] Registering TS with master...
15:12:00.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.889231 4214 heartbeater.cc:510] Master 127.3.105.124:46531 requested a full tablet report, sending...
15:12:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.890187 4216 heartbeater.cc:346] Connected to a master server at 127.3.105.126:35551
15:12:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.890489 4216 heartbeater.cc:463] Registering TS with master...
15:12:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.891135 4216 heartbeater.cc:510] Master 127.3.105.126:35551 requested a full tablet report, sending...
15:12:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.891680 3694 ts_manager.cc:194] Registered new tserver with Master: d74e98e0e83b41619390a0b8f78b6edc (127.3.105.67:34201)
15:12:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.893296 3694 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.105.67:52415
15:12:00.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.893537 3539 ts_manager.cc:194] Registered new tserver with Master: d74e98e0e83b41619390a0b8f78b6edc (127.3.105.67:34201)
15:12:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.894932 4215 heartbeater.cc:346] Connected to a master server at 127.3.105.125:34619
15:12:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.895221 4215 heartbeater.cc:463] Registering TS with master...
15:12:00.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.895888 4215 heartbeater.cc:510] Master 127.3.105.125:34619 requested a full tablet report, sending...
15:12:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.898730 3614 ts_manager.cc:194] Registered new tserver with Master: d74e98e0e83b41619390a0b8f78b6edc (127.3.105.67:34201)
15:12:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:00.905757 3493 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
15:12:00.916 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:12:00.916 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:12:00.917 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.105.126"
port: 35551
}
bound_http_address {
host: "127.3.105.126"
port: 34423
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.105.125"
port: 34619
}
bound_http_address {
host: "127.3.105.125"
port: 42917
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.105.124"
port: 46531
}
bound_http_address {
host: "127.3.105.124"
port: 36795
}
}
}
15:12:00.917 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:12:00.918 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.105.65"
port: 36005
}
bound_http_address {
host: "127.3.105.65"
port: 34741
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.105.66"
port: 41419
}
bound_http_address {
host: "127.3.105.66"
port: 44247
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.105.67"
port: 34201
}
bound_http_address {
host: "127.3.105.67"
port: 39997
}
}
}
15:12:00.918 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:12:00.919 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6344203626747878722/krb5kdc/krb5cc
15:12:00.920 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:12:00.920 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskZW0gWW/test-tmp
15:12:00.921 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:12:00.921 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskZW0gWW/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:12:00.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:12:01.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:01.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:12:01.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:01.244531 4225 mini_oidc.cc:150] Starting JWKS server
15:12:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:01.247790 4225 webserver.cc:458] Webserver started at http://127.0.0.1:45767/ using document root <none> and password file <none>
15:12:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:01.248204 4225 mini_oidc.cc:186] Starting OIDC Discovery server
15:12:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:01.249981 4225 webserver.cc:458] Webserver started at http://127.0.0.1:37335/ using document root <none> and password file <none>
15:12:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:12:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/principal' for realm 'KRBTEST.COM',
15:12:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:12:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:01 dist-test-slave-xd94 krb5kdc[4236](info): setting up network...
15:12:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:12:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:01 dist-test-slave-xd94 krb5kdc[4236](info): set up 2 sockets
15:12:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:01 dist-test-slave-xd94 krb5kdc[4236](info): commencing operation
15:12:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:12:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:01.896661 4214 heartbeater.cc:502] Master 127.3.105.124:46531 was elected leader, sending a full tablet report...
15:12:02.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.264426 4225 mini_kdc.cc:121] Time spent starting KDC: real 0.977s user 0.002s sys 0.014s
15:12:02.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:12:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:12:02.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:12:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:12:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:12:02.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:12:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:12:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:12:02.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:12:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:12:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/test-user.keytab.
15:12:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/test-user.keytab.
15:12:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:02 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694722, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:12:02.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:12:02Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:12:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:12:02Z Disabled control of system clock
15:12:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.32.126@KRBTEST.COM; defaulting to no policy
15:12:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.32.126@KRBTEST.COM" created.
15:12:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab.
15:12:02.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab.
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:02.443826 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/logs
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.126:39801
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.126
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.126
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:02.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.737690 4250 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.738206 4250 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:02.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.738837 4250 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.770875 4250 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.771104 4250 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.771395 4250 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.771618 4250 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:02.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.771807 4250 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.772022 4250 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:02.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.772181 4250 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:02.809216 4250 master_runner.cc:386] Master server non-default flags:
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.126
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.126:39801
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.126
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/logs
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:02.810532 4250 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:02.811563 4250 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.825286 4257 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.825852 4258 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:02.827855 4250 server_base.cc:1034] running on GCE node
15:12:03.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:02.826649 4260 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:03.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.948597 4250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.951143 4250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.952510 4250 hybrid_clock.cc:648] HybridClock initialized: now 1736694723952485 us; error 62 us; skew 500 ppm
15:12:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.953322 4250 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:03.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:03 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694723, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.959830 4250 init.cc:377] Logged in from keytab as kudu/127.4.32.126@KRBTEST.COM (short username kudu)
15:12:03.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.964601 4250 webserver.cc:458] Webserver started at http://127.4.32.126:43961/ using document root <none> and password file <none>
15:12:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.965804 4250 fs_manager.cc:362] Metadata directory not provided
15:12:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.966079 4250 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.966648 4250 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:12:03.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.971799 4250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/instance:
15:12:03.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:03.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:03 on dist-test-slave-xd94"
15:12:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.972846 4250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal/instance:
15:12:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:03 on dist-test-slave-xd94"
15:12:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.980067 4250 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.004s
15:12:03.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.985247 4268 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.986337 4250 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.004s
15:12:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.986614 4250 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:03.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:03 on dist-test-slave-xd94"
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:03.986908 4250 fs_report.cc:389] FS layout report
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/data
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:03.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:04.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.057933 4250 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.063460 4250 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:04.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.063877 4250 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.135039 4250 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.126:39801
15:12:04.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.135092 4320 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.126:39801 every 8 connection(s)
15:12:04.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.137707 4250 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.143020 4322 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:12:04.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.143496 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4250
15:12:04.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.144018 4225 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal/instance
15:12:04.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.157215 4322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
15:12:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.32.125@KRBTEST.COM; defaulting to no policy
15:12:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:04.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.32.125@KRBTEST.COM" created.
15:12:04.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:04 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694723, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab.
15:12:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab.
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.189231 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/logs
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.125:45861
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.125
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.125
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:04.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.196527 4322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } has no permanent_uuid. Determining permanent_uuid...
15:12:04.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.198932 4270 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.202934 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.252367 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 1
15:12:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.256307 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.324661 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 2
15:12:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.328269 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.419613 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 3
15:12:04.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.423082 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.497890 4328 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.498502 4328 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.499202 4328 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:04.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.529580 4328 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:04.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.529852 4328 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.530133 4328 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.530331 4328 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.530519 4328 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.530699 4328 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:04.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.530840 4328 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.566418 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 4
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.567651 4328 master_runner.cc:386] Master server non-default flags:
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.125
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.125:45861
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.125
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/logs
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:04.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:04.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.569322 4328 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:04.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.570169 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:04.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.570542 4328 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:04.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.585084 4338 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.869509 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 5
15:12:04.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.873201 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.420745 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 6
15:12:05.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:05.427701 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.125:45861: Network error: Client connection negotiation failed: client connection to 127.4.32.125:45861: connect: Connection refused (error 111)
15:12:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.585927 4336 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:04.586237 4335 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:04.587576 4328 server_base.cc:1034] running on GCE node
15:12:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.699864 4328 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.702522 4328 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.703938 4328 hybrid_clock.cc:648] HybridClock initialized: now 1736694725703893 us; error 52 us; skew 500 ppm
15:12:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.704777 4328 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:05.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:05 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694725, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.710237 4328 init.cc:377] Logged in from keytab as kudu/127.4.32.125@KRBTEST.COM (short username kudu)
15:12:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.715068 4328 webserver.cc:458] Webserver started at http://127.4.32.125:39695/ using document root <none> and password file <none>
15:12:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.715971 4328 fs_manager.cc:362] Metadata directory not provided
15:12:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.716161 4328 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.716539 4328 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:12:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.721297 4328 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/instance:
15:12:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1"
15:12:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:05 on dist-test-slave-xd94"
15:12:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.722348 4328 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal/instance:
15:12:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1"
15:12:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:05 on dist-test-slave-xd94"
15:12:05.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.729264 4328 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
15:12:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.734529 4349 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.735592 4328 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.735886 4328 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1"
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:05 on dist-test-slave-xd94"
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.736224 4328 fs_report.cc:389] FS layout report
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/data
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:05.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:05.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.795473 4328 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.800786 4328 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.801205 4328 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.869803 4328 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.125:45861
15:12:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.869886 4403 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.125:45861 every 8 connection(s)
15:12:05.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.872336 4328 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:05.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.877403 4404 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:12:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.880178 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4328
15:12:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.880656 4225 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal/instance
15:12:05.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.891160 4404 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
15:12:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.32.124@KRBTEST.COM; defaulting to no policy
15:12:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:05.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.32.124@KRBTEST.COM" created.
15:12:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:05 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694725, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:12:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab.
15:12:05.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.32.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab.
15:12:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.921527 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/logs
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.124:35935
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.124
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.124
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:05.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:05.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.924481 4404 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } has no permanent_uuid. Determining permanent_uuid...
15:12:05.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:05 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694725, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for kudu/127.4.32.125@KRBTEST.COM
15:12:05.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:05.946517 4404 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } has no permanent_uuid. Determining permanent_uuid...
15:12:05.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:05.948884 4353 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:05.952726 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.002171 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 1
15:12:06.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.006067 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.074462 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 2
15:12:06.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.078015 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.169368 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 3
15:12:06.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.173328 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.239634 4410 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.240191 4410 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:06.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.240784 4410 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.271870 4410 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.272101 4410 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.272369 4410 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.272565 4410 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.272753 4410 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.272974 4410 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.273142 4410 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.310362 4410 master_runner.cc:386] Master server non-default flags:
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.124
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.124:35935
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.124
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/logs
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:06.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.311664 4410 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:06.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.312618 4410 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.316705 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 4
15:12:06.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.320852 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.327361 4418 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.488240 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } attempt: 7
15:12:06.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:06 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694723, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for kudu/127.4.32.125@KRBTEST.COM
15:12:06.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.511117 4322 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } has no permanent_uuid. Determining permanent_uuid...
15:12:06.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.516178 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.574671 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 1
15:12:06.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.578590 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.620270 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 5
15:12:06.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.623893 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.659940 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 2
15:12:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.664093 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.749606 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 3
15:12:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.754463 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.894884 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 4
15:12:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.899438 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.171445 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 6
15:12:07.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.178628 4404 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.182832 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 5
15:12:07.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.187103 4322 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.328079 4421 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:06.328357 4419 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:06.329144 4410 server_base.cc:1034] running on GCE node
15:12:07.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.447505 4410 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:07.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.450143 4410 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.451475 4410 hybrid_clock.cc:648] HybridClock initialized: now 1736694727451435 us; error 58 us; skew 500 ppm
15:12:07.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.452279 4410 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:07.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694727, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.457846 4410 init.cc:377] Logged in from keytab as kudu/127.4.32.124@KRBTEST.COM (short username kudu)
15:12:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.462695 4410 webserver.cc:458] Webserver started at http://127.4.32.124:37013/ using document root <none> and password file <none>
15:12:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.463856 4410 fs_manager.cc:362] Metadata directory not provided
15:12:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.464139 4410 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:07.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.464630 4410 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:12:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.469929 4410 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/instance:
15:12:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9927fa28be2142b9ad7e50f2e88470e5"
15:12:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:07 on dist-test-slave-xd94"
15:12:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.471122 4410 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal/instance:
15:12:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9927fa28be2142b9ad7e50f2e88470e5"
15:12:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:07 on dist-test-slave-xd94"
15:12:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.478438 4410 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
15:12:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.483798 4435 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.484941 4410 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:12:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.485291 4410 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9927fa28be2142b9ad7e50f2e88470e5"
15:12:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:07 on dist-test-slave-xd94"
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.485638 4410 fs_report.cc:389] FS layout report
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/data
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:07.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.544075 4410 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.549727 4410 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.550221 4410 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.616586 4410 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.124:35935
15:12:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.616664 4487 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.124:35935 every 8 connection(s)
15:12:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.619165 4410 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.624030 4488 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:12:07.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.628283 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4410
15:12:07.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.628751 4225 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal/instance
15:12:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.640139 4488 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } has no permanent_uuid. Determining permanent_uuid...
15:12:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694727, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.124@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694722, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.32.124@KRBTEST.COM
15:12:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.676110 4488 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } has no permanent_uuid. Determining permanent_uuid...
15:12:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694727, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.124@KRBTEST.COM for kudu/127.4.32.125@KRBTEST.COM
15:12:07.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694722, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.32.125@KRBTEST.COM
15:12:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.697965 4488 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } has no permanent_uuid. Determining permanent_uuid...
15:12:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694727, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.124@KRBTEST.COM for kudu/127.4.32.124@KRBTEST.COM
15:12:07.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694722, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:07.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.724768 4488 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Bootstrap starting.
15:12:07.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.729717 4225 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
15:12:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.731240 4488 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Neither blocks nor log segments found. Creating new log.
15:12:07.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.732982 4488 log.cc:826] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Log is configured to *not* fsync() on all Append() calls
15:12:07.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:07.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:12:07.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.737613 4488 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: No bootstrap required, opened a new log
15:12:07.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:12:07.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:12:07.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.32.126"
port: 39801
}
bound_http_address {
host: "127.4.32.126"
port: 43961
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.32.125"
port: 45861
}
bound_http_address {
host: "127.4.32.125"
port: 39695
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.32.124"
port: 35935
}
bound_http_address {
host: "127.4.32.124"
port: 37013
}
}
}
15:12:07.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:12:07.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:07.739 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:12:07.740 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:12:07.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.739473 4322 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 6
15:12:07.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694723, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for kudu/127.4.32.124@KRBTEST.COM
15:12:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.771010 4488 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.771826 4322 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Bootstrap starting.
15:12:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.772126 4488 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:12:07.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.772444 4488 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9927fa28be2142b9ad7e50f2e88470e5, State: Initialized, Role: FOLLOWER
15:12:07.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:07 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694727, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:07.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:12:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.773310 4488 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:07.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.780225 4322 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Neither blocks nor log segments found. Creating new log.
15:12:07.782 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.782512 4322 log.cc:826] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Log is configured to *not* fsync() on all Append() calls
15:12:07.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.782915 4501 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:07.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.783828 4501 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: FOLLOWER
15:12:07.784 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc
15:12:07.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.785032 4488 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: configured and running, proceeding with master startup.
15:12:07.785 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:12:07.785 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc
15:12:07.786 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:12:07.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.788887 4322 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: No bootstrap required, opened a new log
15:12:07.790 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:12:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.805230 4442 tablet.cc:2367] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:12:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.811563 4322 raft_consensus.cc:357] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.812588 4322 raft_consensus.cc:383] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:12:07.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.812672 4513 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:12:07.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.812976 4513 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:12:07.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.813047 4322 raft_consensus.cc:738] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f533eb645fa24e6aad34fb2d31d04789, State: Initialized, Role: FOLLOWER
15:12:07.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.814005 4322 consensus_queue.cc:260] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:07.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.816622 4515 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.818208 4515 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.818498 4322 sys_catalog.cc:564] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: configured and running, proceeding with master startup.
15:12:07.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.821820 4274 tablet.cc:2367] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:12:07.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.808463 4410 master_runner.cc:186] Error getting master registration for 127.4.32.126:39801: OK, instance_id { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" instance_seqno: 1736694724117137 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:12:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.836671 4410 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:12:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.863667 4526 catalog_manager.cc:1559] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:12:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:07.864023 4526 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:12:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.846508 4250 master_runner.cc:186] Error getting master registration for 127.4.32.125:45861: OK, instance_id { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" instance_seqno: 1736694725849971 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:12:07.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:07.879236 4250 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:12:08.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.070118 4501 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:12:08.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.071977 4501 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:08.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.084105 4501 leader_election.cc:290] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801), 04a13daa6ed24f82baeaaa6d8b3ec6a1 (127.4.32.125:45861)
15:12:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.084503 4296 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9927fa28be2142b9ad7e50f2e88470e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f533eb645fa24e6aad34fb2d31d04789" is_pre_election: true
15:12:08.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.085500 4296 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9927fa28be2142b9ad7e50f2e88470e5 in term 0.
15:12:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.087260 4437 leader_election.cc:304] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9927fa28be2142b9ad7e50f2e88470e5, f533eb645fa24e6aad34fb2d31d04789; no voters:
15:12:08.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.088486 4501 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:12:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.088954 4501 raft_consensus.cc:491] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:12:08.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.089406 4501 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 0 FOLLOWER]: Advancing to term 1
15:12:08.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.096458 4501 raft_consensus.cc:513] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:08.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.099826 4296 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9927fa28be2142b9ad7e50f2e88470e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:08.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.100430 4296 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 0 FOLLOWER]: Advancing to term 1
15:12:08.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.107477 4296 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9927fa28be2142b9ad7e50f2e88470e5 in term 1.
15:12:08.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.108897 4437 leader_election.cc:304] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 9927fa28be2142b9ad7e50f2e88470e5, f533eb645fa24e6aad34fb2d31d04789; no voters:
15:12:08.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.116845 4501 leader_election.cc:290] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [CANDIDATE]: Term 1 election: Requested vote from peers f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801), 04a13daa6ed24f82baeaaa6d8b3ec6a1 (127.4.32.125:45861)
15:12:08.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.119801 4501 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 FOLLOWER]: Leader election won for term 1
15:12:08.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.120880 4501 raft_consensus.cc:695] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 LEADER]: Becoming Leader. State: Replica: 9927fa28be2142b9ad7e50f2e88470e5, State: Running, Role: LEADER
15:12:08.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.122447 4501 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.155756 4528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9927fa28be2142b9ad7e50f2e88470e5. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.159819 4528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: LEADER
15:12:08.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.174763 4530 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:12:08.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.194238 4530 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:12:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.216492 4296 raft_consensus.cc:1270] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Refusing update from remote peer 9927fa28be2142b9ad7e50f2e88470e5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:12:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.218417 4501 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:12:08.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.239091 4404 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } attempt: 7
15:12:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.251088 4515 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9927fa28be2142b9ad7e50f2e88470e5. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.251973 4515 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.266748 4528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.267563 4528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: LEADER
15:12:08.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.275468 4515 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.276226 4515 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:08 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694725, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for kudu/127.4.32.124@KRBTEST.COM
15:12:08.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.288437 4530 catalog_manager.cc:1348] Generated new cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:08.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.288749 4530 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:12:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.330634 4530 catalog_manager.cc:1371] Generated new certificate authority record
15:12:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.341480 4530 catalog_manager.cc:1505] Loading token signing keys...
15:12:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.355073 4404 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Bootstrap starting.
15:12:08.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.370766 4530 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Generated new TSK 0
15:12:08.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.375885 4530 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:12:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.380672 4404 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Neither blocks nor log segments found. Creating new log.
15:12:08.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.383224 4404 log.cc:826] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Log is configured to *not* fsync() on all Append() calls
15:12:08.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.395438 4404 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: No bootstrap required, opened a new log
15:12:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.428071 4404 raft_consensus.cc:357] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.428965 4404 raft_consensus.cc:383] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:12:08.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.429260 4404 raft_consensus.cc:738] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04a13daa6ed24f82baeaaa6d8b3ec6a1, State: Initialized, Role: FOLLOWER
15:12:08.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.430318 4404 consensus_queue.cc:260] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.436527 4404 sys_catalog.cc:564] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: configured and running, proceeding with master startup.
15:12:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.436686 4377 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9927fa28be2142b9ad7e50f2e88470e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" is_pre_election: true
15:12:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.437543 4377 raft_consensus.cc:2463] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9927fa28be2142b9ad7e50f2e88470e5 in term 0.
15:12:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.438823 4375 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 0 FOLLOWER]: Advancing to term 1
15:12:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.443650 4376 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9927fa28be2142b9ad7e50f2e88470e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1"
15:12:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.451651 4375 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Refusing update from remote peer 9927fa28be2142b9ad7e50f2e88470e5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:12:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.434670 4539 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.453855 4528 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:12:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.454499 4539 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: FOLLOWER
15:12:08.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.528477 4547 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.529438 4547 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: FOLLOWER
15:12:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.531870 4539 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9927fa28be2142b9ad7e50f2e88470e5. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.535126 4539 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: FOLLOWER
15:12:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.568605 4555 catalog_manager.cc:1260] Loaded cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.568871 4555 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: loading cluster ID for follower catalog manager: success
15:12:08.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.574489 4555 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: acquiring CA information for follower catalog manager: success
15:12:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.578305 4555 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:12:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.871454 4526 catalog_manager.cc:1260] Loaded cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.871955 4526 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: loading cluster ID for follower catalog manager: success
15:12:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.880085 4526 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: acquiring CA information for follower catalog manager: success
15:12:08.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.887454 4526 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:12:08.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.911219 4528 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9927fa28be2142b9ad7e50f2e88470e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:08.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:08.911980 4528 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: LEADER
15:12:10.791 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.32.125:45861
15:12:10.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:12:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:10.792488 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4328
15:12:10.815 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:10.815 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.32.126:39801
15:12:10.815 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:12:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:10.815768 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4250
15:12:10.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:10.852 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.32.124:35935
15:12:10.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:12:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:10.853363 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4410
15:12:10.879 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:10.879 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.32.125:45861
15:12:10.879 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:12:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:10.881424 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/logs
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:10.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.125:45861
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.125
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39695
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.125
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:10.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:10.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.219867 4563 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.220423 4563 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:11.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.221025 4563 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.252504 4563 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.252749 4563 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.253075 4563 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.253285 4563 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.253584 4563 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.253793 4563 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:11.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.253937 4563 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:11.292896 4563 master_runner.cc:386] Master server non-default flags:
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.125.keytab
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.125
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.125:45861
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.125
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39695
15:12:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/logs
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:11.294173 4563 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:11.295171 4563 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.309232 4571 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.309919 4570 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:11.311836 4563 server_base.cc:1034] running on GCE node
15:12:12.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:11.311095 4573 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:12.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.520309 4563 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:12.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.522825 4563 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:12.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.524168 4563 hybrid_clock.cc:648] HybridClock initialized: now 1736694732524125 us; error 48 us; skew 500 ppm
15:12:12.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.525007 4563 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:12 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694732, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.530666 4563 init.cc:377] Logged in from keytab as kudu/127.4.32.125@KRBTEST.COM (short username kudu)
15:12:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.535868 4563 webserver.cc:458] Webserver started at http://127.4.32.125:39695/ using document root <none> and password file <none>
15:12:12.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.536787 4563 fs_manager.cc:362] Metadata directory not provided
15:12:12.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.537051 4563 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:12.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.545509 4563 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
15:12:12.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.549729 4582 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:12.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.550704 4563 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
15:12:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.551054 4563 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1"
15:12:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:05 on dist-test-slave-xd94"
15:12:12.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.553371 4563 fs_report.cc:389] FS layout report
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/wal
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/data
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:12.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.603250 4563 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.608913 4563 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:12.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.609416 4563 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.675832 4563 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.125:45861
15:12:12.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.675951 4634 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.125:45861 every 8 connection(s)
15:12:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.678602 4563 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-1/data/info.pb
15:12:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.685576 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4563
15:12:12.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:12.686 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.32.126:39801
15:12:12.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.688041 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/logs
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.126:39801
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.126
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43961
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.126
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:12.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.689069 4635 sys_catalog.cc:263] Verifying existing consensus state
15:12:12.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.694491 4635 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Bootstrap starting.
15:12:12.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.742784 4635 log.cc:826] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Log is configured to *not* fsync() on all Append() calls
15:12:12.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.761168 4635 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:12:12.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.762151 4635 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Bootstrap complete.
15:12:12.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.793627 4635 raft_consensus.cc:357] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.796290 4635 raft_consensus.cc:738] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04a13daa6ed24f82baeaaa6d8b3ec6a1, State: Initialized, Role: FOLLOWER
15:12:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.797195 4635 consensus_queue.cc:260] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.800009 4639 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.800974 4639 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: FOLLOWER
15:12:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.801802 4635 sys_catalog.cc:564] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: configured and running, proceeding with master startup.
15:12:12.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.833652 4650 catalog_manager.cc:1260] Loaded cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:12.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.834156 4650 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: loading cluster ID for follower catalog manager: success
15:12:12.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.841915 4650 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: acquiring CA information for follower catalog manager: success
15:12:12.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.847786 4650 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:12:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.831948 4563 master_runner.cc:186] Error getting master registration for 127.4.32.126:39801: Network error: Client connection negotiation failed: client connection to 127.4.32.126:39801: connect: Connection refused (error 111),
15:12:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:12.862442 4563 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:12:13.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.047139 4637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:13.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.047814 4637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:13.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.048503 4637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.096800 4637 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.097121 4637 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.097592 4637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.097934 4637 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.098264 4637 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.098583 4637 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.098832 4637 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:13.144227 4637 master_runner.cc:386] Master server non-default flags:
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.126.keytab
15:12:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.126
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.126:39801
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.126
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43961
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/logs
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:13.145779 4637 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:13.146937 4637 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:13.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.162487 4658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.201370 4210 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1ea0 after lost signal to thread 4083
15:12:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.202534 4210 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcea0 after lost signal to thread 4213
15:12:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.330192 4664 raft_consensus.cc:491] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:12:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.331079 4664 raft_consensus.cc:513] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.348959 4664 leader_election.cc:290] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801), 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935)
15:12:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:14.359647 4584 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.32.126:39801: connect: Connection refused (error 111)
15:12:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:14.369839 4586 leader_election.cc:336] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935): Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:14.372912 4584 leader_election.cc:336] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801): Network error: Client connection negotiation failed: client connection to 127.4.32.126:39801: connect: Connection refused (error 111)
15:12:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.373626 4584 leader_election.cc:304] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 04a13daa6ed24f82baeaaa6d8b3ec6a1; no voters: 9927fa28be2142b9ad7e50f2e88470e5, f533eb645fa24e6aad34fb2d31d04789
15:12:14.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.374605 4664 raft_consensus.cc:2743] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:12:14.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.163792 4661 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:14.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:13.162995 4659 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:14.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:14.422535 4660 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:12:14.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.422669 4637 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:12:14.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.430459 4637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:14.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.433746 4637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:14.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.435179 4637 hybrid_clock.cc:648] HybridClock initialized: now 1736694734435061 us; error 144 us; skew 500 ppm
15:12:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.436395 4637 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:14.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:14 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694734, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:14.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.452446 4637 init.cc:377] Logged in from keytab as kudu/127.4.32.126@KRBTEST.COM (short username kudu)
15:12:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.473448 4637 webserver.cc:458] Webserver started at http://127.4.32.126:43961/ using document root <none> and password file <none>
15:12:14.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.474521 4637 fs_manager.cc:362] Metadata directory not provided
15:12:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.474830 4637 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:14.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.483009 4637 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
15:12:14.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.487965 4673 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.489126 4637 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:12:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.489557 4637 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:14.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:03 on dist-test-slave-xd94"
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.491858 4637 fs_report.cc:389] FS layout report
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/wal
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/data
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:14.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.556293 4637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.563971 4637 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.564535 4637 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:14.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.664723 4637 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.126:39801
15:12:14.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.664824 4725 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.126:39801 every 8 connection(s)
15:12:14.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.667681 4637 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-0/data/info.pb
15:12:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.672865 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4637
15:12:14.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:14.673 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.32.124:35935
15:12:14.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.675446 4225 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:12:14.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/logs
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.124:35935
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.124
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37013
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.124
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:14.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:12:14.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.683678 4726 sys_catalog.cc:263] Verifying existing consensus state
15:12:14.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.697350 4726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Bootstrap starting.
15:12:14.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.749951 4726 log.cc:826] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Log is configured to *not* fsync() on all Append() calls
15:12:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.782959 4726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:12:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.784113 4726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: Bootstrap complete.
15:12:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.821655 4726 raft_consensus.cc:357] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.825237 4726 raft_consensus.cc:738] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f533eb645fa24e6aad34fb2d31d04789, State: Initialized, Role: FOLLOWER
15:12:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.826715 4726 consensus_queue.cc:260] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.848642 4730 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:14.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.849551 4730 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:14.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.851792 4726 sys_catalog.cc:564] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: configured and running, proceeding with master startup.
15:12:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:14.961172 4742 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:12:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:14 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694734, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:15 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694734, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.126@KRBTEST.COM for kudu/127.4.32.125@KRBTEST.COM
15:12:15.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.068471 4637 master_runner.cc:186] Error getting master registration for 127.4.32.124:35935: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111),
15:12:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.097985 4637 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:12:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.126852 3645 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5800 after lost signal to thread 3580
15:12:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.127595 3645 debug-util.cc:398] Leaking SignalData structure 0x7b08000a3f40 after lost signal to thread 3648
15:12:15.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.161415 4728 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:12:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.162367 4728 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:12:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.163432 4728 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:12:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.219650 4728 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:12:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.219970 4728 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:12:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.220429 4728 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:12:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.220759 4728 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:12:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.221084 4728 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:12:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.221388 4728 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:12:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.221622 4728 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45767/jwks.json
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.286177 4728 master_runner.cc:386] Master server non-default flags:
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.32.84:46591
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.32.126:39801,127.4.32.125:45861,127.4.32.124:35935
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/krb5kdc/kudu_127.4.32.124.keytab
15:12:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.32.124
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.32.124:35935
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45767/jwks.json
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.32.124
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37013
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/logs
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Jan 2025 14:43:21 UTC on a633016ff5c9
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4144
15:12:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:12:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.287886 4728 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:12:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.289149 4728 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:12:15.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.323621 4752 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:15.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.909982 4757 raft_consensus.cc:491] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:12:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.910511 4757 raft_consensus.cc:513] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:15.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.913508 4757 leader_election.cc:290] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801), 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935)
15:12:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:15 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694732, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for kudu/127.4.32.126@KRBTEST.COM
15:12:15.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.941823 4586 leader_election.cc:336] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935): Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.946478 4701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f533eb645fa24e6aad34fb2d31d04789" is_pre_election: true
15:12:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.947479 4701 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 04a13daa6ed24f82baeaaa6d8b3ec6a1 in term 1.
15:12:15.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.948729 4584 leader_election.cc:304] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 04a13daa6ed24f82baeaaa6d8b3ec6a1, f533eb645fa24e6aad34fb2d31d04789; no voters: 9927fa28be2142b9ad7e50f2e88470e5
15:12:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.949702 4757 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:12:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.950073 4757 raft_consensus.cc:491] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:12:15.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.950405 4757 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 1 FOLLOWER]: Advancing to term 2
15:12:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.960551 4757 raft_consensus.cc:513] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.963500 4701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "f533eb645fa24e6aad34fb2d31d04789"
15:12:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.964056 4701 raft_consensus.cc:3054] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 1 FOLLOWER]: Advancing to term 2
15:12:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.967456 4586 leader_election.cc:336] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935): Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111)
15:12:15.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.968087 4757 leader_election.cc:290] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 election: Requested vote from peers f533eb645fa24e6aad34fb2d31d04789 (127.4.32.126:39801), 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935)
15:12:15.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.976040 4701 raft_consensus.cc:2463] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 04a13daa6ed24f82baeaaa6d8b3ec6a1 in term 2.
15:12:15.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.977320 4584 leader_election.cc:304] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 04a13daa6ed24f82baeaaa6d8b3ec6a1, f533eb645fa24e6aad34fb2d31d04789; no voters: 9927fa28be2142b9ad7e50f2e88470e5
15:12:15.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.978502 4757 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 2 FOLLOWER]: Leader election won for term 2
15:12:15.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.979790 4742 catalog_manager.cc:1260] Loaded cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:15.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.980036 4742 catalog_manager.cc:1553] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: loading cluster ID for follower catalog manager: success
15:12:15.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.983850 4757 raft_consensus.cc:695] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [term 2 LEADER]: Becoming Leader. State: Replica: 04a13daa6ed24f82baeaaa6d8b3ec6a1, State: Running, Role: LEADER
15:12:15.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.984916 4757 consensus_queue.cc:237] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.988430 4742 catalog_manager.cc:1575] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: acquiring CA information for follower catalog manager: success
15:12:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.007288 4742 catalog_manager.cc:1603] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:12:16.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.008469 4763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 04a13daa6ed24f82baeaaa6d8b3ec6a1. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:16.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.009382 4763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: LEADER
15:12:16.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.027489 4765 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
15:12:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.035591 4765 catalog_manager.cc:1485] Initializing Kudu cluster ID...
15:12:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.039973 4765 catalog_manager.cc:1260] Loaded cluster ID: 2979ffbe480c4180b657e76d9377763f
15:12:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.040390 4765 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
15:12:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.045187 4765 catalog_manager.cc:1505] Loading token signing keys...
15:12:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.049436 4765 catalog_manager.cc:5910] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1: Loaded TSK: 0
15:12:16.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.050791 4765 catalog_manager.cc:1515] Initializing in-progress tserver states...
15:12:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.513397 4701 raft_consensus.cc:1270] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [term 2 FOLLOWER]: Refusing update from remote peer 04a13daa6ed24f82baeaaa6d8b3ec6a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:12:16.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.515549 4763 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:12:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:16.558597 4586 consensus_peers.cc:487] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 -> Peer 9927fa28be2142b9ad7e50f2e88470e5 (127.4.32.124:35935): Couldn't send request to peer 9927fa28be2142b9ad7e50f2e88470e5. Status: Network error: Client connection negotiation failed: client connection to 127.4.32.124:35935: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:12:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.568346 4766 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 04a13daa6ed24f82baeaaa6d8b3ec6a1. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.569525 4766 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.604847 4766 sys_catalog.cc:455] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:16.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.605558 4766 sys_catalog.cc:458] T 00000000000000000000000000000000 P f533eb645fa24e6aad34fb2d31d04789 [sys.catalog]: This master's current role is: FOLLOWER
15:12:16.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.608736 4763 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.609463 4763 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a13daa6ed24f82baeaaa6d8b3ec6a1 [sys.catalog]: This master's current role is: LEADER
15:12:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:15.333848 4728 server_base.cc:1034] running on GCE node
15:12:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.334329 4755 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:16.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:15.330981 4751 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:12:16.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.672585 4728 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:12:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.679920 4728 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:12:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.695698 4728 hybrid_clock.cc:648] HybridClock initialized: now 1736694736695646 us; error 48 us; skew 500 ppm
15:12:16.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.696841 4728 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:12:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:16 dist-test-slave-xd94 krb5kdc[4236](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694736, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:12:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.704622 4728 init.cc:377] Logged in from keytab as kudu/127.4.32.124@KRBTEST.COM (short username kudu)
15:12:16.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.713138 4728 webserver.cc:458] Webserver started at http://127.4.32.124:37013/ using document root <none> and password file <none>
15:12:16.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.714416 4728 fs_manager.cc:362] Metadata directory not provided
15:12:16.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.714790 4728 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:12:16.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.726025 4728 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.001s sys 0.004s
15:12:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.732599 4779 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:12:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.734033 4728 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
15:12:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.734432 4728 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data,/tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9927fa28be2142b9ad7e50f2e88470e5"
15:12:16.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-12 15:12:07 on dist-test-slave-xd94"
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.737846 4728 fs_report.cc:389] FS layout report
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/wal
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/data
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:12:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:12:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:12:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:12:16.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:12:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.816454 4728 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:12:16.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.824389 4728 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:12:16.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.824987 4728 kserver.cc:163] Server-wide thread pool size limit: 3276
15:12:16.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.925460 4728 rpc_server.cc:307] RPC server started. Bound to: 127.4.32.124:35935
15:12:16.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.925601 4830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.4.32.124:35935 every 8 connection(s)
15:12:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.929448 4728 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskZW0gWW/test-tmp/mini-kudu-cluster6867728240452652323/master-2/data/info.pb
15:12:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.931078 4225 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu as pid 4728
15:12:16.931 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:12:16.936 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.32.125:45861(127.4.32.125:45861)] connecting to peer
15:12:16.938 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.32.126:39801(127.4.32.126:39801)] connecting to peer
15:12:16.941 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.32.124:35935(127.4.32.124:35935)] connecting to peer
15:12:16.949 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.32.125:45861(127.4.32.125:45861)] Successfully connected to peer
15:12:16.950 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.32.124:35935(127.4.32.124:35935)] Successfully connected to peer
15:12:16.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.944720 4831 sys_catalog.cc:263] Verifying existing consensus state
15:12:16.953 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.32.126:39801(127.4.32.126:39801)] Successfully connected to peer
15:12:16.969 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:12:16.970 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.32.125
15:12:16.974 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.32.124
15:12:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:16.971657 4831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Bootstrap starting.
15:12:16.981 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.32.126
15:12:17.033 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.32.126:39801(127.4.32.126:39801)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.033 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.32.126:39801: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:17.037134 4838 negotiation.cc:337] Failed RPC negotiation. Trace:
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.953114 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:59922 (local address 127.4.32.126:39801)
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980115 (+ 27001us) server_negotiation.cc:206] Beginning negotiation
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980143 (+ 28us) server_negotiation.cc:399] Waiting for connection header
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980193 (+ 50us) server_negotiation.cc:407] Connection header received
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980331 (+ 138us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980360 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980566 (+ 206us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.980737 (+ 171us) server_negotiation.cc:226] Negotiated authn=SASL
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.985511 (+ 4774us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.987735 (+ 2224us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.026383 (+ 38648us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.026989 (+ 606us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.035152 (+ 8163us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:59922: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:59922): Cannot send after transport endpoint shutdown (error 108)
15:12:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20453,"thread_start_us":21630,"threads_started":1}
15:12:17.050 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.32.126:39801(127.4.32.126:39801)] handling channelInactive
15:12:17.053 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.32.124:35935(127.4.32.124:35935)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.32.124:35935: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:17.057277 4837 negotiation.cc:337] Failed RPC negotiation. Trace:
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.950211 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37616 (local address 127.4.32.124:35935)
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.960693 (+ 10482us) server_negotiation.cc:206] Beginning negotiation
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.960734 (+ 41us) server_negotiation.cc:399] Waiting for connection header
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.960812 (+ 78us) server_negotiation.cc:407] Connection header received
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.967213 (+ 6401us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.967279 (+ 66us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.967763 (+ 484us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.968180 (+ 417us) server_negotiation.cc:226] Negotiated authn=SASL
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.000947 (+ 32767us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.003996 (+ 3049us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.053270 (+ 49274us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.054012 (+ 742us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
15:12:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.055324 (+ 1312us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37616: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
15:12:17.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":9746,"thread_start_us":9477,"threads_started":1}
15:12:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.061707 4831 log.cc:826] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Log is configured to *not* fsync() on all Append() calls
15:12:17.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 12 15:12:17 dist-test-slave-xd94 krb5kdc[4236](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736694732, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.32.125@KRBTEST.COM for kudu/127.4.32.124@KRBTEST.COM
15:12:17.086 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.32.125:45861(127.4.32.125:45861)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.086 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.32.125:45861: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
15:12:17.086 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=154), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.125:45861, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.126:39801, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.124:35935, [98ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.126:39801: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.124:35935: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [151ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.125:45861: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15:12:17.089 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=157), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.125:45861, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.126:39801, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.32.124:35935, [98ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.126:39801: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [118ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.124:35935: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [151ms] Sub RPC ConnectToMaster: received response from server master-127.4.32.125:45861: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
15:12:17.089 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.32.126:39801(127.4.32.126:39801)] disconnecting while in state TERMINATED
15:12:17.090 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.32.125:45861(127.4.32.125:45861)] disconnecting while in state TERMINATED
15:12:17.090 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.32.124:35935(127.4.32.124:35935)] disconnecting while in state TERMINATED
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.093220 4831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250112 15:12:17.093422 4836 negotiation.cc:337] Failed RPC negotiation. Trace:
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.949512 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35796 (local address 127.4.32.125:45861)
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.966549 (+ 17037us) server_negotiation.cc:206] Beginning negotiation
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.966578 (+ 29us) server_negotiation.cc:399] Waiting for connection header
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.966634 (+ 56us) server_negotiation.cc:407] Connection header received
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.966780 (+ 146us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.966827 (+ 47us) server_negotiation.cc:461] Received NEGOTIATE request from client
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:16.967075 (+ 248us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.007506 (+ 40431us) server_negotiation.cc:226] Negotiated authn=SASL
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.007948 (+ 442us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.009662 (+ 1714us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.078496 (+ 68834us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
15:12:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.079221 (+ 725us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
15:12:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0112 15:12:17.091754 (+ 12533us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35796: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:35796): Cannot send after transport endpoint shutdown (error 108)
15:12:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16414,"thread_start_us":17843,"threads_started":1}
15:12:17.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.094189 4831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5: Bootstrap complete.
15:12:17.095 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.32.124:35935(127.4.32.124:35935)] handling channelInactive
15:12:17.098 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:12:17.100 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:12:17.100 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.32.125:45861(127.4.32.125:45861)] handling channelInactive
15:12:17.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.127329 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4637
15:12:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.146165 4831 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.149746 4831 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9927fa28be2142b9ad7e50f2e88470e5, State: Initialized, Role: FOLLOWER
15:12:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.150709 4831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } }
15:12:17.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.158070 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4563
15:12:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.158696 4831 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: configured and running, proceeding with master startup.
15:12:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.159502 4806 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [term 1 FOLLOWER]: Advancing to term 2
15:12:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.157074 4846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.167259 4846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: FOLLOWER
15:12:17.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.182693 4858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:17.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.183558 4858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: FOLLOWER
15:12:17.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.189218 4846 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 04a13daa6ed24f82baeaaa6d8b3ec6a1. Latest consensus state: current_term: 2 leader_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f533eb645fa24e6aad34fb2d31d04789" member_type: VOTER last_known_addr { host: "127.4.32.126" port: 39801 } } peers { permanent_uuid: "04a13daa6ed24f82baeaaa6d8b3ec6a1" member_type: VOTER last_known_addr { host: "127.4.32.125" port: 45861 } } peers { permanent_uuid: "9927fa28be2142b9ad7e50f2e88470e5" member_type: VOTER last_known_addr { host: "127.4.32.124" port: 35935 } } }
15:12:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.189850 4846 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9927fa28be2142b9ad7e50f2e88470e5 [sys.catalog]: This master's current role is: FOLLOWER
15:12:17.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:17.192524 4225 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4728
15:12:18.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:12:18Z chronyd exiting
15:12:19.320 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:12:19.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.321967 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 3775
15:12:19.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.340476 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 3936
15:12:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.358788 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 4080
15:12:19.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.380259 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 3505
15:12:19.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.401719 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 3576
15:12:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250112 15:12:19.423997 3493 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskZW0gWW/build/tsan/bin/kudu with pid 3651
15:12:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-12T15:12:20Z chronyd exiting
15:12:21.496 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
15:12:21.676 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: giving up after 1 attempts
15:12:21.676 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.4.32.125:45861,127.4.32.126:39801,127.4.32.124:35935). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more