21:09:05.342 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:09:05.342 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskOzYfYq/test-tmp
21:09:05.342 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:09:05.342 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskOzYfYq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:09:05.343 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:09:05.649 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:09:05.649 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:09:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:05.652644 14955 mini_oidc.cc:150] Starting JWKS server
21:09:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:05.655839 14955 webserver.cc:458] Webserver started at http://127.0.0.1:37285/ using document root <none> and password file <none>
21:09:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:05.656227 14955 mini_oidc.cc:186] Starting OIDC Discovery server
21:09:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:05.657820 14955 webserver.cc:458] Webserver started at http://127.0.0.1:38813/ using document root <none> and password file <none>
21:09:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:09:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:09:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:09:05Z Disabled control of system clock
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:05.715003 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/logs
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data/info.pb
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.254:40167
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.254
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json with env {}
21:09:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:05.997756 14967 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:05.998317 14967 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:05.998838 14967 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.029603 14967 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:06.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.029871 14967 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.030081 14967 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.030267 14967 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.030503 14967 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:06.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.030671 14967 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37285/jwks.json
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:06.064523 14967 master_runner.cc:386] Master server non-default flags:
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.254:40167
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data/info.pb
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.254
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/logs
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:06.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:06.065758 14967 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:06.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:06.067226 14967 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:06.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.081138 14973 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.081468 14974 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:06.081900 14976 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:07.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:06.082989 14967 server_base.cc:1034] running on GCE node
21:09:07.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.167584 14967 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:07.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.170050 14967 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:07.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.171396 14967 hybrid_clock.cc:648] HybridClock initialized: now 1736543347171349 us; error 48 us; skew 500 ppm
21:09:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.172190 14967 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:07.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.182929 14967 webserver.cc:458] Webserver started at http://127.14.154.254:42479/ using document root <none> and password file <none>
21:09:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.183887 14967 fs_manager.cc:362] Metadata directory not provided
21:09:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.184114 14967 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.184607 14967 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.189416 14967 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data/instance:
21:09:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efc9933fff1d4ccbbf585b30e8223eaa"
21:09:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:07 on dist-test-slave-4hht"
21:09:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.190537 14967 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal/instance:
21:09:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efc9933fff1d4ccbbf585b30e8223eaa"
21:09:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:07 on dist-test-slave-4hht"
21:09:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.197355 14967 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
21:09:07.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.202445 14983 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:07.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.203384 14967 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.203689 14967 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "efc9933fff1d4ccbbf585b30e8223eaa"
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:07 on dist-test-slave-4hht"
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.204035 14967 fs_report.cc:389] FS layout report
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data/data
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:07.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:07.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.250510 14967 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.251926 14967 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:07.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.252380 14967 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:07.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.316188 14967 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.254:40167
21:09:07.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.316259 15034 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.254:40167 every 8 connection(s)
21:09:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.318783 14967 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/data/info.pb
21:09:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.323630 15035 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.323585 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 14967
21:09:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.323930 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-0/wal/instance
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.327134 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/logs
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data/info.pb
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:07.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.253:43485
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.253
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:07.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json with env {}
21:09:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.335871 15035 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
21:09:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.360271 15035 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } has no permanent_uuid. Determining permanent_uuid...
21:09:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.362572 14985 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.366226 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:07.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.415748 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 1
21:09:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.419404 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.487824 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 2
21:09:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.491477 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:07.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.582918 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 3
21:09:07.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.586593 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:07.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.621459 15037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.622046 15037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.622539 15037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.652786 15037 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.653057 15037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.653264 15037 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.653460 15037 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.653650 15037 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:07.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.653803 15037 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37285/jwks.json
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.687997 15037 master_runner.cc:386] Master server non-default flags:
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.253:43485
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data/info.pb
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.253
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/logs
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:07.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.689294 15037 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:07.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.690917 15037 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.705020 15046 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.730019 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 4
21:09:07.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.733697 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:08.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.033031 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 5
21:09:08.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:08.037160 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:08.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.584527 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 6
21:09:08.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:08.590482 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.253:43485: Network error: Client connection negotiation failed: client connection to 127.14.154.253:43485: connect: Connection refused (error 111)
21:09:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.705399 15049 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:07.705827 15047 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:07.707044 15037 server_base.cc:1034] running on GCE node
21:09:08.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.789024 15037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:08.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.791435 15037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:08.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.792752 15037 hybrid_clock.cc:648] HybridClock initialized: now 1736543348792707 us; error 61 us; skew 500 ppm
21:09:08.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.793545 15037 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:08.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.799398 15037 webserver.cc:458] Webserver started at http://127.14.154.253:38521/ using document root <none> and password file <none>
21:09:08.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.800238 15037 fs_manager.cc:362] Metadata directory not provided
21:09:08.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.800417 15037 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:08.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.801007 15037 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.805694 15037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data/instance:
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "45c48ee07a1e4f448a53ac3caaa7316e"
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:08 on dist-test-slave-4hht"
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.806694 15037 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal/instance:
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "45c48ee07a1e4f448a53ac3caaa7316e"
21:09:08.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:08 on dist-test-slave-4hht"
21:09:08.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.813395 15037 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
21:09:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.818471 15058 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.819381 15037 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.819681 15037 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "45c48ee07a1e4f448a53ac3caaa7316e"
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:08 on dist-test-slave-4hht"
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.819998 15037 fs_report.cc:389] FS layout report
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal
21:09:08.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data/data
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.876056 15037 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.877504 15037 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.877959 15037 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.942363 15037 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.253:43485
21:09:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.942462 15109 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.253:43485 every 8 connection(s)
21:09:08.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.944976 15037 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/data/info.pb
21:09:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.948949 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15037
21:09:08.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.949414 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-1/wal/instance
21:09:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.950584 15110 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.952572 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/logs
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data/info.pb
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.252:33999
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.252
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:08.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json with env {}
21:09:08.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.963903 15110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
21:09:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.983997 15110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } has no permanent_uuid. Determining permanent_uuid...
21:09:08.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:08.997202 15110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } has no permanent_uuid. Determining permanent_uuid...
21:09:09.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:08.999681 15062 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.003593 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.053009 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 1
21:09:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.056785 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.125129 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 2
21:09:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.129128 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.220505 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 3
21:09:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.224339 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.250805 15112 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:09.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.251372 15112 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.251953 15112 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.281193 15112 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.281512 15112 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:09.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.281782 15112 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.282055 15112 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.282320 15112 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.282519 15112 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37285/jwks.json
21:09:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.315550 15112 master_runner.cc:386] Master server non-default flags:
21:09:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.252:33999
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37285/jwks.json
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data/info.pb
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.252
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/logs
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:09.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.317065 15112 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.318630 15112 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:09.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.333689 15123 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.367710 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 4
21:09:09.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.371459 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.650846 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } attempt: 7
21:09:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.661691 15035 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } has no permanent_uuid. Determining permanent_uuid...
21:09:09.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.666392 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.670738 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 5
21:09:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.674646 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.724875 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 1
21:09:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.728420 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.809787 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 2
21:09:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.813264 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.898756 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 3
21:09:09.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.902734 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.043193 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 4
21:09:10.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.047632 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.221982 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 6
21:09:10.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.228026 15110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.335440 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 5
21:09:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.339002 15035 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.14.154.252:33999: Network error: Client connection negotiation failed: client connection to 127.14.154.252:33999: connect: Connection refused (error 111)
21:09:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.334152 15122 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:09.334579 15125 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:09.334965 15112 server_base.cc:1034] running on GCE node
21:09:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.436503 15112 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.438861 15112 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.440176 15112 hybrid_clock.cc:648] HybridClock initialized: now 1736543350440151 us; error 40 us; skew 500 ppm
21:09:10.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.440945 15112 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.447036 15112 webserver.cc:458] Webserver started at http://127.14.154.252:34991/ using document root <none> and password file <none>
21:09:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.447889 15112 fs_manager.cc:362] Metadata directory not provided
21:09:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.448061 15112 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:10.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.448441 15112 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.453332 15112 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data/instance:
21:09:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d3162a1ad47442ba5fd49e277dbccff"
21:09:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:10 on dist-test-slave-4hht"
21:09:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.454308 15112 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal/instance:
21:09:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d3162a1ad47442ba5fd49e277dbccff"
21:09:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:10 on dist-test-slave-4hht"
21:09:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.460796 15112 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
21:09:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.465912 15137 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.467079 15112 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
21:09:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.467435 15112 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal
21:09:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1d3162a1ad47442ba5fd49e277dbccff"
21:09:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:10 on dist-test-slave-4hht"
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.467800 15112 fs_report.cc:389] FS layout report
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data/data
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:10.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.517129 15112 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.518433 15112 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:10.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.518805 15112 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:10.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.581655 15112 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.252:33999
21:09:10.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.581729 15188 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.252:33999 every 8 connection(s)
21:09:10.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.584229 15112 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/data/info.pb
21:09:10.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.588994 15189 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.594379 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15112
21:09:10.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.594749 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/master-2/wal/instance
21:09:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.600692 15189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } has no permanent_uuid. Determining permanent_uuid...
21:09:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.630112 15189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } has no permanent_uuid. Determining permanent_uuid...
21:09:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.638842 15189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } has no permanent_uuid. Determining permanent_uuid...
21:09:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.652686 15189 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: Bootstrap starting.
21:09:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.658435 15189 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: Neither blocks nor log segments found. Creating new log.
21:09:10.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.660183 15189 log.cc:826] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: Log is configured to *not* fsync() on all Append() calls
21:09:10.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.665114 15189 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: No bootstrap required, opened a new log
21:09:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.685457 15189 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.686082 15189 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.686344 15189 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1d3162a1ad47442ba5fd49e277dbccff, State: Initialized, Role: FOLLOWER
21:09:10.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.687274 15189 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.689935 15198 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.690574 15198 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: This master's current role is: FOLLOWER
21:09:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.691354 15189 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: configured and running, proceeding with master startup.
21:09:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.713968 15209 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:09:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.714486 15209 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:09:10.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.709750 15112 master_runner.cc:186] Error getting master registration for 127.14.154.254:40167: OK, instance_id { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" instance_seqno: 1736543347298700 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:09:10.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.741897 15112 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:10.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.773394 15143 tablet.cc:2367] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:09:10.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.891332 15035 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 6
21:09:10.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.910080 15035 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: Bootstrap starting.
21:09:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.917915 15035 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: Neither blocks nor log segments found. Creating new log.
21:09:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.920217 15035 log.cc:826] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: Log is configured to *not* fsync() on all Append() calls
21:09:10.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.925051 15035 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: No bootstrap required, opened a new log
21:09:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.956259 15035 raft_consensus.cc:357] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:10.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.957401 15035 raft_consensus.cc:383] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:10.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.957835 15035 raft_consensus.cc:738] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: efc9933fff1d4ccbbf585b30e8223eaa, State: Initialized, Role: FOLLOWER
21:09:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.958782 15035 consensus_queue.cc:260] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.962105 15213 sys_catalog.cc:455] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.963021 15213 sys_catalog.cc:458] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: This master's current role is: FOLLOWER
21:09:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.964224 15035 sys_catalog.cc:564] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: configured and running, proceeding with master startup.
21:09:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.993052 15224 catalog_manager.cc:1559] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:09:10.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:10.993376 15224 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:09:11.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:11.010193 14989 tablet.cc:2367] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:09:11.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:10.991714 14967 master_runner.cc:186] Error getting master registration for 127.14.154.253:43485: OK, instance_id { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" instance_seqno: 1736543348924373 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:09:11.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.013622 14967 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.134127 15198 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.134706 15198 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.138079 15198 leader_election.cc:290] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers efc9933fff1d4ccbbf585b30e8223eaa (127.14.154.254:40167), 45c48ee07a1e4f448a53ac3caaa7316e (127.14.154.253:43485)
21:09:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.138391 15010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d3162a1ad47442ba5fd49e277dbccff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" is_pre_election: true
21:09:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.139222 15010 raft_consensus.cc:2463] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1d3162a1ad47442ba5fd49e277dbccff in term 0.
21:09:11.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.140806 15138 leader_election.cc:304] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d3162a1ad47442ba5fd49e277dbccff, efc9933fff1d4ccbbf585b30e8223eaa; no voters:
21:09:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.141844 15198 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Leader pre-election won for term 1
21:09:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.142182 15198 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.142481 15198 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 0 FOLLOWER]: Advancing to term 1
21:09:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.149940 15198 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:11.152984 15031 debug-util.cc:398] Leaking SignalData structure 0x7b0800073a80 after lost signal to thread 14968
21:09:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.155220 15010 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d3162a1ad47442ba5fd49e277dbccff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "efc9933fff1d4ccbbf585b30e8223eaa"
21:09:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.155519 15198 leader_election.cc:290] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [CANDIDATE]: Term 1 election: Requested vote from peers efc9933fff1d4ccbbf585b30e8223eaa (127.14.154.254:40167), 45c48ee07a1e4f448a53ac3caaa7316e (127.14.154.253:43485)
21:09:11.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.155786 15010 raft_consensus.cc:3054] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 0 FOLLOWER]: Advancing to term 1
21:09:11.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.166390 15010 raft_consensus.cc:2463] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1d3162a1ad47442ba5fd49e277dbccff in term 1.
21:09:11.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.167498 15138 leader_election.cc:304] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1d3162a1ad47442ba5fd49e277dbccff, efc9933fff1d4ccbbf585b30e8223eaa; no voters:
21:09:11.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.168344 15198 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 1 FOLLOWER]: Leader election won for term 1
21:09:11.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.180447 15198 raft_consensus.cc:695] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [term 1 LEADER]: Becoming Leader. State: Replica: 1d3162a1ad47442ba5fd49e277dbccff, State: Running, Role: LEADER
21:09:11.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.183009 15198 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:11.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.209601 15226 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1d3162a1ad47442ba5fd49e277dbccff. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.212687 15226 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: This master's current role is: LEADER
21:09:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.226464 15228 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:11.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.252867 15228 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:11.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.288532 15110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } attempt: 7
21:09:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.296325 15010 raft_consensus.cc:1270] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [term 1 FOLLOWER]: Refusing update from remote peer 1d3162a1ad47442ba5fd49e277dbccff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:09:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.299010 15198 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [LEADER]: Connected to new peer: Peer: permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:09:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.343226 15110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: Bootstrap starting.
21:09:11.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.350339 15110 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: Neither blocks nor log segments found. Creating new log.
21:09:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.350494 15213 sys_catalog.cc:455] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1d3162a1ad47442ba5fd49e277dbccff. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.351249 15213 sys_catalog.cc:458] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: This master's current role is: FOLLOWER
21:09:11.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.353004 15110 log.cc:826] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: Log is configured to *not* fsync() on all Append() calls
21:09:11.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.358465 15229 mvcc.cc:204] Tried to move back new op lower bound from 7112881566892359680 to 7112881566472114176. Current Snapshot: MvccSnapshot[applied={T|T < 7112881566892359680}]
21:09:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.361748 15226 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.362457 15226 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: This master's current role is: LEADER
21:09:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.369690 15213 sys_catalog.cc:455] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.370375 15213 sys_catalog.cc:458] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa [sys.catalog]: This master's current role is: FOLLOWER
21:09:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.372174 15228 catalog_manager.cc:1348] Generated new cluster ID: 60547ed598454c878cd8b9a1d49d600e
21:09:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.372462 15228 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:11.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.384228 15110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: No bootstrap required, opened a new log
21:09:11.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.414929 15228 catalog_manager.cc:1371] Generated new certificate authority record
21:09:11.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.420517 15228 catalog_manager.cc:1505] Loading token signing keys...
21:09:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.463723 15110 raft_consensus.cc:357] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.464792 15110 raft_consensus.cc:383] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:11.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.465099 15110 raft_consensus.cc:738] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45c48ee07a1e4f448a53ac3caaa7316e, State: Initialized, Role: FOLLOWER
21:09:11.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.466009 15110 consensus_queue.cc:260] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } }
21:09:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.483878 15228 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff: Generated new TSK 0
21:09:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.486645 15228 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.500372 15110 sys_catalog.cc:564] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: configured and running, proceeding with master startup.
21:09:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.500751 15085 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d3162a1ad47442ba5fd49e277dbccff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" is_pre_election: true
21:09:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.502166 15084 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1d3162a1ad47442ba5fd49e277dbccff" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45c48ee07a1e4f448a53ac3caaa7316e"
21:09:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.506111 15083 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [term 0 FOLLOWER]: Advancing to term 1
21:09:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.512657 15083 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [term 1 FOLLOWER]: Refusing update from remote peer 1d3162a1ad47442ba5fd49e277dbccff: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:09:11.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.516852 15236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.517835 15236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: This master's current role is: FOLLOWER
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.529912 15198 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [LEADER]: Connected to new peer: Peer: permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.530419 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data
21:09:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/logs
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data/info.pb
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:11.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.193:0
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.193
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.193
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:11.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:09:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.584152 15236 sys_catalog.cc:455] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1d3162a1ad47442ba5fd49e277dbccff. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.584877 15236 sys_catalog.cc:458] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: This master's current role is: FOLLOWER
21:09:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.586092 15244 sys_catalog.cc:455] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.586977 15244 sys_catalog.cc:458] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e [sys.catalog]: This master's current role is: FOLLOWER
21:09:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.649367 15253 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:09:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.910499 15198 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1d3162a1ad47442ba5fd49e277dbccff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "efc9933fff1d4ccbbf585b30e8223eaa" member_type: VOTER last_known_addr { host: "127.14.154.254" port: 40167 } } peers { permanent_uuid: "45c48ee07a1e4f448a53ac3caaa7316e" member_type: VOTER last_known_addr { host: "127.14.154.253" port: 43485 } } peers { permanent_uuid: "1d3162a1ad47442ba5fd49e277dbccff" member_type: VOTER last_known_addr { host: "127.14.154.252" port: 33999 } } }
21:09:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.911306 15198 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1d3162a1ad47442ba5fd49e277dbccff [sys.catalog]: This master's current role is: LEADER
21:09:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.999018 15224 catalog_manager.cc:1260] Loaded cluster ID: 60547ed598454c878cd8b9a1d49d600e
21:09:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:11.999478 15224 catalog_manager.cc:1553] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: loading cluster ID for follower catalog manager: success
21:09:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.006901 15224 catalog_manager.cc:1575] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: acquiring CA information for follower catalog manager: success
21:09:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.017405 15224 catalog_manager.cc:1603] T 00000000000000000000000000000000 P efc9933fff1d4ccbbf585b30e8223eaa: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:12.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.074764 15238 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:12.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.075611 15238 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.076493 15238 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:12.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.133862 15238 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.135666 15238 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.154.193
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.197269 15238 tablet_server_runner.cc:78] Tablet server non-default flags:
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:12.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.193:0
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data/info.pb
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.193
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.193
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/logs
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:12.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:12.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.200004 15238 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.202229 15238 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:12.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.242997 15260 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.677711 15253 catalog_manager.cc:1260] Loaded cluster ID: 60547ed598454c878cd8b9a1d49d600e
21:09:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.678021 15253 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: loading cluster ID for follower catalog manager: success
21:09:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.697582 15253 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: acquiring CA information for follower catalog manager: success
21:09:12.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.715960 15253 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 45c48ee07a1e4f448a53ac3caaa7316e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:13.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:12.251368 15238 server_base.cc:1034] running on GCE node
21:09:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.251513 15259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:13.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:12.247939 15263 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.882675 15238 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.885434 15238 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.886957 15238 hybrid_clock.cc:648] HybridClock initialized: now 1736543353886921 us; error 41 us; skew 500 ppm
21:09:13.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.888114 15238 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.901648 15238 webserver.cc:458] Webserver started at http://127.14.154.193:45235/ using document root <none> and password file <none>
21:09:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.902793 15238 fs_manager.cc:362] Metadata directory not provided
21:09:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.903051 15238 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:13.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.903553 15238 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:13.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.910398 15238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data/instance:
21:09:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f99ad94a064d4c98a45b8e5c39c8c748"
21:09:13.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:13 on dist-test-slave-4hht"
21:09:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.912456 15238 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal/instance:
21:09:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f99ad94a064d4c98a45b8e5c39c8c748"
21:09:13.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:13 on dist-test-slave-4hht"
21:09:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.921429 15238 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
21:09:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.928442 15270 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.929754 15238 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:09:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.930172 15238 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal
21:09:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f99ad94a064d4c98a45b8e5c39c8c748"
21:09:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:13 on dist-test-slave-4hht"
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:13.930840 15238 fs_report.cc:389] FS layout report
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data/data
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:13.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.039449 15238 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.040939 15238 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.041347 15238 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:14.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.044080 15238 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:09:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.048458 15238 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:09:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.048676 15238 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
21:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.048877 15238 ts_tablet_manager.cc:610] Registered 0 tablets
21:09:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.049011 15238 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:09:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.202610 15238 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.193:36137
21:09:14.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.202715 15383 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.193:36137 every 8 connection(s)
21:09:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.205196 15238 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/data/info.pb
21:09:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.212972 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15238
21:09:14.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.213446 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-0/wal/instance
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.227612 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/logs
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data/info.pb
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.194:0
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.194
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.194
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:14.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:14.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:09:14.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.253495 15386 heartbeater.cc:346] Connected to a master server at 127.14.154.254:40167
21:09:14.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.254009 15386 heartbeater.cc:463] Registering TS with master...
21:09:14.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.255034 15385 heartbeater.cc:346] Connected to a master server at 127.14.154.253:43485
21:09:14.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.255352 15385 heartbeater.cc:463] Registering TS with master...
21:09:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.255322 15386 heartbeater.cc:510] Master 127.14.154.254:40167 requested a full tablet report, sending...
21:09:14.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.256532 15385 heartbeater.cc:510] Master 127.14.154.253:43485 requested a full tablet report, sending...
21:09:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.259898 15075 ts_manager.cc:194] Registered new tserver with Master: f99ad94a064d4c98a45b8e5c39c8c748 (127.14.154.193:36137)
21:09:14.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.260093 15000 ts_manager.cc:194] Registered new tserver with Master: f99ad94a064d4c98a45b8e5c39c8c748 (127.14.154.193:36137)
21:09:14.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.261631 15384 heartbeater.cc:346] Connected to a master server at 127.14.154.252:33999
21:09:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.261969 15384 heartbeater.cc:463] Registering TS with master...
21:09:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.262666 15384 heartbeater.cc:510] Master 127.14.154.252:33999 requested a full tablet report, sending...
21:09:14.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.264673 15154 ts_manager.cc:194] Registered new tserver with Master: f99ad94a064d4c98a45b8e5c39c8c748 (127.14.154.193:36137)
21:09:14.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.266638 15154 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.154.193:56051
21:09:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.574034 15394 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.574626 15394 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.575161 15394 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.622943 15394 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.623827 15394 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.154.194
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.657783 15394 tablet_server_runner.cc:78] Tablet server non-default flags:
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.194:0
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data/info.pb
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.194
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.194
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/logs
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:09:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:14.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:14.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:14.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.659396 15394 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:14.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:14.661134 15394 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:14.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.678524 15401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.269660 15384 heartbeater.cc:502] Master 127.14.154.252:33999 was elected leader, sending a full tablet report...
21:09:15.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.678998 15403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:15.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:14.679921 15400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:15.885820 15402 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1205 milliseconds
21:09:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.885924 15394 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.887090 15394 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:15.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.889499 15394 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:15.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.890893 15394 hybrid_clock.cc:648] HybridClock initialized: now 1736543355890867 us; error 39 us; skew 500 ppm
21:09:15.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.891638 15394 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:15.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.897287 15394 webserver.cc:458] Webserver started at http://127.14.154.194:43313/ using document root <none> and password file <none>
21:09:15.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.898087 15394 fs_manager.cc:362] Metadata directory not provided
21:09:15.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.898263 15394 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:15.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.898666 15394 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.903460 15394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data/instance:
21:09:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aca4a9f52a91472687b4c1fdcb292577"
21:09:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:15 on dist-test-slave-4hht"
21:09:15.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.904443 15394 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal/instance:
21:09:15.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aca4a9f52a91472687b4c1fdcb292577"
21:09:15.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:15 on dist-test-slave-4hht"
21:09:15.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.910950 15394 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
21:09:15.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.915923 15411 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.916963 15394 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.917269 15394 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aca4a9f52a91472687b4c1fdcb292577"
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:15 on dist-test-slave-4hht"
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.917558 15394 fs_report.cc:389] FS layout report
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal
21:09:15.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data/data
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:15.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.970938 15394 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:15.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.972280 15394 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:15.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.972687 15394 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:15.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.975338 15394 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:09:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.980819 15394 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:09:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.981097 15394 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:09:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.981390 15394 ts_tablet_manager.cc:610] Registered 0 tablets
21:09:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:15.981590 15394 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:09:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.114892 15394 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.194:43351
21:09:16.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.114996 15524 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.194:43351 every 8 connection(s)
21:09:16.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.117317 15394 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/data/info.pb
21:09:16.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.119063 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15394
21:09:16.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.119661 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-1/wal/instance
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.135890 14955 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/logs
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data/info.pb
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.195:0
21:09:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.195
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.195
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:09:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.167390 15525 heartbeater.cc:346] Connected to a master server at 127.14.154.252:33999
21:09:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.167876 15525 heartbeater.cc:463] Registering TS with master...
21:09:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.168936 15525 heartbeater.cc:510] Master 127.14.154.252:33999 requested a full tablet report, sending...
21:09:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.170482 15526 heartbeater.cc:346] Connected to a master server at 127.14.154.253:43485
21:09:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.170830 15526 heartbeater.cc:463] Registering TS with master...
21:09:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.171540 15154 ts_manager.cc:194] Registered new tserver with Master: aca4a9f52a91472687b4c1fdcb292577 (127.14.154.194:43351)
21:09:16.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.171670 15526 heartbeater.cc:510] Master 127.14.154.253:43485 requested a full tablet report, sending...
21:09:16.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.173399 15154 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.154.194:38383
21:09:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.174877 15075 ts_manager.cc:194] Registered new tserver with Master: aca4a9f52a91472687b4c1fdcb292577 (127.14.154.194:43351)
21:09:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.177196 15527 heartbeater.cc:346] Connected to a master server at 127.14.154.254:40167
21:09:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.177479 15527 heartbeater.cc:463] Registering TS with master...
21:09:16.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.178045 15527 heartbeater.cc:510] Master 127.14.154.254:40167 requested a full tablet report, sending...
21:09:16.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.180419 15000 ts_manager.cc:194] Registered new tserver with Master: aca4a9f52a91472687b4c1fdcb292577 (127.14.154.194:43351)
21:09:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.446946 15532 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.447367 15532 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.447786 15532 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.477897 15532 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.478652 15532 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.154.195
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.513088 15532 tablet_server_runner.cc:78] Tablet server non-default flags:
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.154.212:46027
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.154.195:0
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data/info.pb
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.154.195
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.154.254:40167,127.14.154.253:43485,127.14.154.252:33999
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:16.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.154.195
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/logs
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:16.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.514613 15532 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.516139 15532 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:16.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.531595 15542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.177670 15525 heartbeater.cc:502] Master 127.14.154.252:33999 was elected leader, sending a full tablet report...
21:09:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.531960 15544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:16.532420 15541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:17.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:16.532938 15532 server_base.cc:1034] running on GCE node
21:09:17.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.611963 15532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.614079 15532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.615398 15532 hybrid_clock.cc:648] HybridClock initialized: now 1736543357615375 us; error 39 us; skew 500 ppm
21:09:17.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.616173 15532 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:17.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.622234 15532 webserver.cc:458] Webserver started at http://127.14.154.195:40497/ using document root <none> and password file <none>
21:09:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.623060 15532 fs_manager.cc:362] Metadata directory not provided
21:09:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.623240 15532 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:17.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.623627 15532 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.628278 15532 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data/instance:
21:09:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a59bda7c8c5a4b6bb311553f1700ce0f"
21:09:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:17 on dist-test-slave-4hht"
21:09:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.629424 15532 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal/instance:
21:09:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a59bda7c8c5a4b6bb311553f1700ce0f"
21:09:17.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:17 on dist-test-slave-4hht"
21:09:17.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.636268 15532 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
21:09:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.641355 15552 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.642282 15532 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
21:09:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.642582 15532 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal
21:09:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a59bda7c8c5a4b6bb311553f1700ce0f"
21:09:17.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:17 on dist-test-slave-4hht"
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.642889 15532 fs_report.cc:389] FS layout report
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data/data
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:17.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.687431 15532 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.688879 15532 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.689332 15532 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:17.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.691743 15532 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:09:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.695698 15532 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:09:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.695904 15532 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:09:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.696179 15532 ts_tablet_manager.cc:610] Registered 0 tablets
21:09:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.696338 15532 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:09:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.822733 15532 rpc_server.cc:307] RPC server started. Bound to: 127.14.154.195:43549
21:09:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.822846 15664 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.154.195:43549 every 8 connection(s)
21:09:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.825188 15532 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/data/info.pb
21:09:17.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.827574 14955 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15532
21:09:17.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.828016 14955 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster8398200663579017935/ts-2/wal/instance
21:09:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.867242 15665 heartbeater.cc:346] Connected to a master server at 127.14.154.252:33999
21:09:17.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.867735 15665 heartbeater.cc:463] Registering TS with master...
21:09:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.869423 15667 heartbeater.cc:346] Connected to a master server at 127.14.154.254:40167
21:09:17.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.869760 15667 heartbeater.cc:463] Registering TS with master...
21:09:17.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.870082 15665 heartbeater.cc:510] Master 127.14.154.252:33999 requested a full tablet report, sending...
21:09:17.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.870913 15667 heartbeater.cc:510] Master 127.14.154.254:40167 requested a full tablet report, sending...
21:09:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.872531 15666 heartbeater.cc:346] Connected to a master server at 127.14.154.253:43485
21:09:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.872855 15154 ts_manager.cc:194] Registered new tserver with Master: a59bda7c8c5a4b6bb311553f1700ce0f (127.14.154.195:43549)
21:09:17.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.873184 15666 heartbeater.cc:463] Registering TS with master...
21:09:17.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.873667 15000 ts_manager.cc:194] Registered new tserver with Master: a59bda7c8c5a4b6bb311553f1700ce0f (127.14.154.195:43549)
21:09:17.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.874155 15666 heartbeater.cc:510] Master 127.14.154.253:43485 requested a full tablet report, sending...
21:09:17.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.874311 15154 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.14.154.195:59397
21:09:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.876832 15075 ts_manager.cc:194] Registered new tserver with Master: a59bda7c8c5a4b6bb311553f1700ce0f (127.14.154.195:43549)
21:09:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:17.890514 14955 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
21:09:17.900 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:09:17.900 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:09:17.901 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.154.254"
port: 40167
}
bound_http_address {
host: "127.14.154.254"
port: 42479
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.154.253"
port: 43485
}
bound_http_address {
host: "127.14.154.253"
port: 38521
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.154.252"
port: 33999
}
bound_http_address {
host: "127.14.154.252"
port: 34991
}
}
}
21:09:17.901 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:09:17.902 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.154.193"
port: 36137
}
bound_http_address {
host: "127.14.154.193"
port: 45235
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.154.194"
port: 43351
}
bound_http_address {
host: "127.14.154.194"
port: 43313
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.154.195"
port: 43549
}
bound_http_address {
host: "127.14.154.195"
port: 40497
}
}
}
21:09:17.902 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:09:17.903 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster5094088018675056070/krb5kdc/krb5cc
21:09:17.904 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:09:17.904 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskOzYfYq/test-tmp
21:09:17.905 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:09:17.905 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskOzYfYq/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:09:17.906 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:09:18.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:18.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:09:18.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:18.231029 15676 mini_oidc.cc:150] Starting JWKS server
21:09:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:18.234370 15676 webserver.cc:458] Webserver started at http://127.0.0.1:38817/ using document root <none> and password file <none>
21:09:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:18.234833 15676 mini_oidc.cc:186] Starting OIDC Discovery server
21:09:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:18.236557 15676 webserver.cc:458] Webserver started at http://127.0.0.1:40545/ using document root <none> and password file <none>
21:09:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:09:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/principal' for realm 'KRBTEST.COM',
21:09:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:09:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:18 dist-test-slave-4hht krb5kdc[15687](info): setting up network...
21:09:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:09:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:18 dist-test-slave-4hht krb5kdc[15687](info): set up 2 sockets
21:09:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:18 dist-test-slave-4hht krb5kdc[15687](info): commencing operation
21:09:18.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:09:18.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:18.877991 15665 heartbeater.cc:502] Master 127.14.154.252:33999 was elected leader, sending a full tablet report...
21:09:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.251850 15676 mini_kdc.cc:121] Time spent starting KDC: real 0.979s user 0.002s sys 0.015s
21:09:19.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:09:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:09:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:09:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:09:19.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:09:19.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:09:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:09:19.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:09:19.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:09:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:09:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/test-user.keytab.
21:09:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/test-user.keytab.
21:09:19.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:19 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543359, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:19.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:09:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:09:19Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:09:19.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:09:19Z Disabled control of system clock
21:09:19.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.79.62@KRBTEST.COM; defaulting to no policy
21:09:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.79.62@KRBTEST.COM" created.
21:09:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab.
21:09:19.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab.
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:19.410915 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:19.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.734130 15701 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:19.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.734722 15701 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.735383 15701 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.766613 15701 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.766860 15701 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.767228 15701 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.767518 15701 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.767808 15701 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.768095 15701 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.768335 15701 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:19.804349 15701 master_runner.cc:386] Master server non-default flags:
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:19.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:19.805886 15701 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:19.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:19.806852 15701 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:19.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.821295 15707 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:20.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:20.449965 15106 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae4a0 after lost signal to thread 15041
21:09:20.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:20.450848 15106 debug-util.cc:398] Leaking SignalData structure 0x7b0800097840 after lost signal to thread 15109
21:09:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.824982 15710 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:19.821861 15708 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:21.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.051550 15709 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1227 milliseconds
21:09:21.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.051651 15701 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:21.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.053014 15701 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.055766 15701 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.057165 15701 hybrid_clock.cc:648] HybridClock initialized: now 1736543361057107 us; error 70 us; skew 500 ppm
21:09:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.057904 15701 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:21 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543361, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.068598 15701 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.074177 15701 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.075205 15701 fs_manager.cc:362] Metadata directory not provided
21:09:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.075417 15701 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.075846 15701 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.080411 15701 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/instance:
21:09:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.081540 15701 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal/instance:
21:09:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:21.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.089476 15701 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
21:09:21.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.095054 15718 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.096045 15701 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:09:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.096335 15701 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:21.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.096668 15701 fs_report.cc:389] FS layout report
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.142957 15701 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.148233 15701 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.148725 15701 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:21.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.232385 15701 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:21.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.232570 15769 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.235066 15701 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:21.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.241288 15770 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:21.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.244417 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15701
21:09:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.244899 15676 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal/instance
21:09:21.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.79.61@KRBTEST.COM; defaulting to no policy
21:09:21.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.257227 15770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } has no permanent_uuid. Determining permanent_uuid...
21:09:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.79.61@KRBTEST.COM" created.
21:09:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab.
21:09:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab.
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:21 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543361, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.287256 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:21.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:21.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.305166 15770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
21:09:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.308136 15721 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.312891 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.362437 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 1
21:09:21.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.366935 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.435353 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 2
21:09:21.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.439779 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.531198 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 3
21:09:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.535578 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.602351 15776 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.602947 15776 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.603590 15776 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.634675 15776 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:21.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.634950 15776 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:21.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.635325 15776 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:21.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.635612 15776 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.635905 15776 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.636162 15776 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.636359 15776 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.673127 15776 master_runner.cc:386] Master server non-default flags:
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:21.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.674687 15776 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:21.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.675704 15776 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:21.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.679000 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 4
21:09:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.683429 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.692191 15783 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:21.982834 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 5
21:09:21.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.987977 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:22.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.535436 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 6
21:09:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:22.541584 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:22.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.696131 15784 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:21.692807 15786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:22.803197 15785 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:09:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.803313 15776 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.807269 15776 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:22.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.809937 15776 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:22.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.811333 15776 hybrid_clock.cc:648] HybridClock initialized: now 1736543362811278 us; error 68 us; skew 500 ppm
21:09:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.812108 15776 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:22 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543362, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:22.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.817351 15776 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.822360 15776 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.823220 15776 fs_manager.cc:362] Metadata directory not provided
21:09:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.823405 15776 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:22.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.823777 15776 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.828297 15776 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/instance:
21:09:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.829375 15776 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal/instance:
21:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:22.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.835973 15776 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.004s
21:09:22.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.840978 15797 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.841898 15776 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.842206 15776 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:22.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.842506 15776 fs_report.cc:389] FS layout report
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.893656 15776 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:22.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.899072 15776 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:22.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.899857 15776 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.966564 15776 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.966620 15848 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.969301 15776 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.974462 15849 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.974834 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15776
21:09:22.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.975312 15676 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal/instance
21:09:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:22.987521 15849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } has no permanent_uuid. Determining permanent_uuid...
21:09:22.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.15.79.60@KRBTEST.COM; defaulting to no policy
21:09:22.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:22.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.15.79.60@KRBTEST.COM" created.
21:09:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:23 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543362, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:23.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:09:23.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab.
21:09:23.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.15.79.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab.
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.016253 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:23.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:23.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.019420 15849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
21:09:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:23 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543362, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:23.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.044665 15849 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } has no permanent_uuid. Determining permanent_uuid...
21:09:23.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.047010 15798 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.050733 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.100190 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 1
21:09:23.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.103682 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.172124 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 2
21:09:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.175911 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.267347 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 3
21:09:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.271261 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.332239 15855 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.332779 15855 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:23.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.333390 15855 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.363415 15855 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:23.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.363624 15855 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.363922 15855 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.364142 15855 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.364338 15855 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.364562 15855 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:23.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.364714 15855 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.399662 15855 master_runner.cc:386] Master server non-default flags:
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:23.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.400889 15855 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.401783 15855 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:23.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.415024 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 4
21:09:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.416522 15863 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.419545 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.602052 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } attempt: 7
21:09:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:23 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543361, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:23.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.623908 15770 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } has no permanent_uuid. Determining permanent_uuid...
21:09:23.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.628883 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.687369 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 1
21:09:23.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.691109 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.718928 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 5
21:09:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.722666 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.772567 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 2
21:09:23.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.776598 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:23.862069 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 3
21:09:23.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.867058 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:24.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.007489 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 4
21:09:24.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.012619 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:24.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.270123 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 6
21:09:24.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.276837 15849 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:24.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.295992 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 5
21:09:24.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.299777 15770 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.419067 15864 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:23.416605 15866 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.515722 15865 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1100 milliseconds
21:09:24.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.516310 15855 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:24.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.519459 15855 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:24.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.522207 15855 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:24.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.523721 15855 hybrid_clock.cc:648] HybridClock initialized: now 1736543364523669 us; error 60 us; skew 500 ppm
21:09:24.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.524755 15855 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.530169 15855 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:24.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.535184 15855 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:24.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.536085 15855 fs_manager.cc:362] Metadata directory not provided
21:09:24.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.536266 15855 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:24.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.536671 15855 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:09:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.541147 15855 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/instance:
21:09:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.542151 15855 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal/instance:
21:09:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:24.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.549454 15855 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
21:09:24.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.555951 15880 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.557121 15855 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:09:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.557477 15855 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.558575 15855 fs_report.cc:389] FS layout report
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:24.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:24.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.611260 15855 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.616235 15855 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:24.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.616654 15855 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.682056 15855 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:24.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.682135 15932 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:24.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.684571 15855 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.689411 15933 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:09:24.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.689525 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 15855
21:09:24.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.689985 15676 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal/instance
21:09:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.703198 15933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } has no permanent_uuid. Determining permanent_uuid...
21:09:24.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543359, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:24.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.734778 15933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } has no permanent_uuid. Determining permanent_uuid...
21:09:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543359, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:24.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:24.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.755623 15933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } has no permanent_uuid. Determining permanent_uuid...
21:09:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543359, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:24.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:24.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.783849 15676 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
21:09:24.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.784947 15933 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.791942 15933 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Neither blocks nor log segments found. Creating new log.
21:09:24.793 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:24.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:09:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.794054 15933 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:24.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:09:24.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:09:24.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.79.62"
port: 39595
}
bound_http_address {
host: "127.15.79.62"
port: 42193
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.79.61"
port: 40625
}
bound_http_address {
host: "127.15.79.61"
port: 39379
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.79.60"
port: 43573
}
bound_http_address {
host: "127.15.79.60"
port: 42079
}
}
}
21:09:24.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:09:24.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:24.796 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:09:24.796 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:09:24.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.798959 15933 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: No bootstrap required, opened a new log
21:09:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:24.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:09:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.820528 15933 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.821125 15933 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:24.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.821385 15933 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Initialized, Role: FOLLOWER
21:09:24.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.822204 15933 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:24.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.824851 15944 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:24.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.825788 15944 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:24.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:24.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.826722 15933 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: configured and running, proceeding with master startup.
21:09:24.827 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:24.828 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.851044 15955 catalog_manager.cc:1559] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:09:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.851794 15955 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:09:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.852319 15770 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 6
21:09:24.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:24 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543361, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:24.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.871249 15887 tablet.cc:2367] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:09:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.846971 15855 master_runner.cc:186] Error getting master registration for 127.15.79.62:39595: OK, instance_id { permanent_uuid: "ad20382b782c44a3a984136f62be401f" instance_seqno: 1736543361210043 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:09:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.876219 15855 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.882468 15770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:24.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.887475 15770 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Neither blocks nor log segments found. Creating new log.
21:09:24.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.889091 15770 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:24.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.893018 15770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: No bootstrap required, opened a new log
21:09:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.915584 15770 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.916417 15770 raft_consensus.cc:383] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.916725 15770 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.917572 15770 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.919869 15959 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.920459 15959 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:24.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.921888 15770 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.947073 15970 catalog_manager.cc:1559] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:09:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:24.947551 15970 catalog_manager.cc:874] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:09:24.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.946954 15701 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: OK, instance_id { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" instance_seqno: 1736543362948045 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:09:24.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:24.970711 15701 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:25.157819 15724 tablet.cc:2367] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:09:25.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.276041 15944 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:25.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.276584 15944 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.280567 15745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e646947c6524250b8154c630cc52ebb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f" is_pre_election: true
21:09:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.281575 15745 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6e646947c6524250b8154c630cc52ebb in term 0.
21:09:25.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.282894 15884 leader_election.cc:304] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6e646947c6524250b8154c630cc52ebb, ad20382b782c44a3a984136f62be401f; no voters:
21:09:25.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.284402 15944 leader_election.cc:290] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625)
21:09:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.284895 15944 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Leader pre-election won for term 1
21:09:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.285228 15944 raft_consensus.cc:491] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.285543 15944 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 0 FOLLOWER]: Advancing to term 1
21:09:25.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.291596 15944 raft_consensus.cc:513] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:25.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.293628 15944 leader_election.cc:290] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [CANDIDATE]: Term 1 election: Requested vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625)
21:09:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.294847 15745 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e646947c6524250b8154c630cc52ebb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f"
21:09:25.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.295362 15745 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 0 FOLLOWER]: Advancing to term 1
21:09:25.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.307610 15745 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6e646947c6524250b8154c630cc52ebb in term 1.
21:09:25.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.308508 15884 leader_election.cc:304] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6e646947c6524250b8154c630cc52ebb, ad20382b782c44a3a984136f62be401f; no voters:
21:09:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.309094 15944 raft_consensus.cc:2798] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 FOLLOWER]: Leader election won for term 1
21:09:25.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.309635 15944 raft_consensus.cc:695] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 LEADER]: Becoming Leader. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Running, Role: LEADER
21:09:25.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.312263 15944 consensus_queue.cc:237] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.324311 15972 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e646947c6524250b8154c630cc52ebb. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.325043 15972 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: LEADER
21:09:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.327638 15974 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.335305 15974 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.337316 15849 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } attempt: 7
21:09:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.359221 15745 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Refusing update from remote peer 6e646947c6524250b8154c630cc52ebb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:09:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.360921 15972 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:09:25.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:25 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543362, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.401690 15849 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.409152 15849 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Neither blocks nor log segments found. Creating new log.
21:09:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.412449 15849 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.428143 15959 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e646947c6524250b8154c630cc52ebb. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.431280 15959 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.431793 15976 mvcc.cc:204] Tried to move back new op lower bound from 7112881624488468480 to 7112881624331730944. Current Snapshot: MvccSnapshot[applied={T|T < 7112881624488468480}]
21:09:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.437336 15944 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.438077 15944 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: LEADER
21:09:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.441201 15974 catalog_manager.cc:1348] Generated new cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.440901 15849 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: No bootstrap required, opened a new log
21:09:25.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.441478 15974 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.475994 15959 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.476799 15959 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:25.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.520723 15849 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.516050 15974 catalog_manager.cc:1371] Generated new certificate authority record
21:09:25.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.522939 15849 raft_consensus.cc:383] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:09:25.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.523775 15849 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:25.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.526077 15849 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:25.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.529563 15974 catalog_manager.cc:1505] Loading token signing keys...
21:09:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.545742 15823 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e646947c6524250b8154c630cc52ebb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.546495 15823 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 0 FOLLOWER]: Advancing to term 1
21:09:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.548718 15824 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6e646947c6524250b8154c630cc52ebb" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" is_pre_election: true
21:09:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.558297 15823 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6e646947c6524250b8154c630cc52ebb in term 1.
21:09:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.559487 15822 raft_consensus.cc:1270] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Refusing update from remote peer 6e646947c6524250b8154c630cc52ebb: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:09:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.561797 15974 catalog_manager.cc:5899] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Generated new TSK 0
21:09:25.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.563618 15974 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.565690 15972 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [LEADER]: Connected to new peer: Peer: permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:09:25.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.577701 15849 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.625564 15982 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.626385 15982 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.623889 15983 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6e646947c6524250b8154c630cc52ebb. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.629160 15983 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.630116 15987 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "6e646947c6524250b8154c630cc52ebb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.631722 15987 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.662642 16001 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:25.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.662912 16001 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.674844 16001 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.680996 16001 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:25.830 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.62:39595
21:09:25.831 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:09:25.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.833338 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15701
21:09:25.865 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:25.865 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.61:40625
21:09:25.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:09:25.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.866505 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15776
21:09:25.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:25.897 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.60:43573
21:09:25.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:09:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.897737 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15855
21:09:25.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:25.918 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.62:39595
21:09:25.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:25.920804 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:25.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:26.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.220913 16003 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:26.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.221449 16003 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:26.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.222038 16003 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.253412 16003 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:26.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.253630 16003 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.253887 16003 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.254089 16003 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.254284 16003 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.254465 16003 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.254606 16003 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:26.289614 16003 master_runner.cc:386] Master server non-default flags:
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:26.290817 16003 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:26.291790 16003 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.308455 16010 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.308652 16011 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:26.310299 16013 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.399487 16012 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:09:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.399542 16003 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:27.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.403177 16003 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.406036 16003 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.407438 16003 hybrid_clock.cc:648] HybridClock initialized: now 1736543367407403 us; error 54 us; skew 500 ppm
21:09:27.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.408216 16003 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:27 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543367, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:27.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.420020 16003 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:27.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.425209 16003 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.426048 16003 fs_manager.cc:362] Metadata directory not provided
21:09:27.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.426240 16003 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:27.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.433954 16003 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
21:09:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.438026 16022 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:27.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.438973 16003 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.439261 16003 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.441541 16003 fs_report.cc:389] FS layout report
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:27.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.487613 16003 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:27.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.492956 16003 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:27.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.493388 16003 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.560070 16003 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.560184 16073 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.562759 16003 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:27.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.572147 16074 sys_catalog.cc:263] Verifying existing consensus state
21:09:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.573407 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16003
21:09:27.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:27.574 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.61:40625
21:09:27.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:09:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.575563 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.578064 16074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.624591 16074 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.643018 16074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.643919 16074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap complete.
21:09:27.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.677395 16074 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.680775 16074 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.681732 16074 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.684309 16079 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.684952 16079 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.686180 16074 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.719003 16090 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.719476 16090 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: success
21:09:27.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.727505 16090 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: acquiring CA information for follower catalog manager: success
21:09:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:27 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543367, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.733356 16090 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:27.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.748365 16003 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111),
21:09:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.778205 16003 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:27.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.924604 16076 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:27.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.925143 16076 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:27.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.925724 16076 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.957276 16076 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.957481 16076 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:27.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.957736 16076 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.957937 16076 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.958122 16076 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.958302 16076 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:27.958438 16076 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.994779 16076 master_runner.cc:386] Master server non-default flags:
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.996037 16076 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:27.997035 16076 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:28.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:28.012966 16102 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:29.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.154515 15380 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7980 after lost signal to thread 15254
21:09:29.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.155596 15380 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1580 after lost signal to thread 15383
21:09:29.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:28.013298 16100 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:29.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:28.014050 16099 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:29.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:28.015404 16076 server_base.cc:1034] running on GCE node
21:09:29.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.177105 16076 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.190960 16076 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.196332 16107 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.196858 16107 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:29.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.199813 16076 hybrid_clock.cc:648] HybridClock initialized: now 1736543369199723 us; error 97 us; skew 500 ppm
21:09:29.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.201009 16076 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.203949 16107 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:29 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543369, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.208777 16076 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.232714 16076 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.234009 16076 fs_manager.cc:362] Metadata directory not provided
21:09:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.234300 16076 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:29.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.238109 16023 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:29.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.244422 16025 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625): Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:29.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.247061 16023 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.248159 16023 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:29.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.249401 16107 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:09:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.250725 16076 fs_manager.cc:714] Time spent opening directory manager: real 0.012s user 0.004s sys 0.008s
21:09:29.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.262826 16116 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.264166 16076 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.001s sys 0.007s
21:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.264604 16076 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.268224 16076 fs_report.cc:389] FS layout report
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:29.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:29.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.318828 16076 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.326023 16076 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:29.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.326550 16076 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:29.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.420042 16076 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:29.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.420100 16167 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:29.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.422892 16076 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:29.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.429194 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16076
21:09:29.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:29.430 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.60:43573
21:09:29.430 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.432132 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.443652 16168 sys_catalog.cc:263] Verifying existing consensus state
21:09:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.452152 16168 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:29.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.507712 16168 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:29.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.532318 16168 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:29.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.533282 16168 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap complete.
21:09:29.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.577519 16168 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.580941 16168 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.582260 16168 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:29.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.595927 16168 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:29.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.591815 16172 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.599031 16172 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:29.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.719178 16183 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.731053 16183 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:29 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543369, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:29.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.761332 16183 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:29.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.777926 16183 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:29.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.794600 16076 master_runner.cc:186] Error getting master registration for 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111),
21:09:29.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.827926 16076 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.917141 16170 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.917753 16170 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.918387 16170 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.955749 16170 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.956008 16170 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.956377 16170 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.956686 16170 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.956959 16170 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.957231 16170 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:29.957432 16170 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.996219 16170 master_runner.cc:386] Master server non-default flags:
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:29.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.997924 16170 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:29.999706 16170 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:30.016690 16193 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:31.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.073611 16197 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.074091 16197 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.082850 16197 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.117612 16201 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.120160 16201 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.135182 16049 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f" is_pre_election: true
21:09:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.135962 16049 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 1.
21:09:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.137488 16119 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters:
21:09:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.138826 16023 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.139441 16201 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:31.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.139936 16201 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Leader pre-election won for term 2
21:09:31.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.140251 16201 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:31.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.140606 16201 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 1 FOLLOWER]: Advancing to term 2
21:09:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.149449 16117 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:31.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.164687 16143 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ad20382b782c44a3a984136f62be401f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" is_pre_election: true
21:09:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.166487 16117 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.171940 16201 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.175855 16143 raft_consensus.cc:2388] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ad20382b782c44a3a984136f62be401f in current term 2: Already voted for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in this term.
21:09:31.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.182917 16025 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.183784 16197 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 1 FOLLOWER]: Advancing to term 2
21:09:31.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.186246 16049 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f"
21:09:31.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.194736 16117 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:31.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.193320 16197 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:09:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.196285 16201 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 election: Requested vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:31.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.201416 16049 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 2.
21:09:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.206727 16119 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.207516 16201 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Leader election won for term 2
21:09:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.208074 16201 raft_consensus.cc:695] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 LEADER]: Becoming Leader. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Running, Role: LEADER
21:09:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.209049 16201 consensus_queue.cc:237] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.231032 16204 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.232064 16204 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.248036 16207 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.260251 16207 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.275156 16207 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.275458 16207 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:31.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.281106 16207 catalog_manager.cc:1505] Loading token signing keys...
21:09:31.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.297092 16207 catalog_manager.cc:5910] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Loaded TSK: 0
21:09:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.299173 16207 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.660452 16117 consensus_peers.cc:487] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 -> Peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Couldn't send request to peer 6e646947c6524250b8154c630cc52ebb. Status: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:09:31.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.783015 16049 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Refusing update from remote peer b42c9a70f6c1438385ef2f0d250f8fc8: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:09:31.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.785141 16204 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:09:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.812579 16208 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.813621 16208 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.843894 16210 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:31.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.844666 16210 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:31.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.848079 16208 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.849212 16208 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.413897 16192 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 16170
21:09:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:30.019652 16194 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.918784 16170 thread.cc:635] GCE (cloud detector) Time spent creating pthread: real 1.902s user 0.663s sys 1.082s
21:09:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.919375 16170 thread.cc:601] GCE (cloud detector) Time spent starting thread: real 1.902s user 0.663s sys 1.082s
21:09:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.927695 16170 server_base.cc:1034] running on GCE node
21:09:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:31.928154 16216 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:31.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.929596 16170 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.932415 16170 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.933866 16170 hybrid_clock.cc:648] HybridClock initialized: now 1736543371933824 us; error 28 us; skew 500 ppm
21:09:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.934918 16170 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:31.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:31 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543371, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:31.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.941955 16170 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:31.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.952262 16170 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.953511 16170 fs_manager.cc:362] Metadata directory not provided
21:09:31.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.953792 16170 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:31.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.963968 16170 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
21:09:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.969221 16222 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.970366 16170 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.970741 16170 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:31.974144 16170 fs_report.cc:389] FS layout report
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:31.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:31.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:32.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.035136 16170 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.040840 16170 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.041340 16170 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.121903 16170 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.122093 16274 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.124867 16170 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:32.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.130666 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16170
21:09:32.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:32.132 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.62:39595(127.15.79.62:39595)] connecting to peer
21:09:32.133 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.61:40625(127.15.79.61:40625)] connecting to peer
21:09:32.135 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.60:43573(127.15.79.60:43573)] connecting to peer
21:09:32.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.137678 16275 sys_catalog.cc:263] Verifying existing consensus state
21:09:32.145 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.79.61:40625(127.15.79.61:40625)] Successfully connected to peer
21:09:32.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.146132 16275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:32.149 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.79.62:39595(127.15.79.62:39595)] Successfully connected to peer
21:09:32.152 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.79.60:43573(127.15.79.60:43573)] Successfully connected to peer
21:09:32.157 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:32.157 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:32.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:32.178 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:32.180 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:32.181 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:32.181 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:32.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:32.185 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:32.185 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:32.186 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.61
21:09:32.205 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.62
21:09:32.223 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:32.224 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:32.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.229772 16275 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:32.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:32.237 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:32.238 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:32.245 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.60
21:09:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.250541 16275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.254774 16275 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap complete.
21:09:32.265 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xd9458614, L:/127.0.0.1:36040 - R:/127.15.79.61:40625] using SASL/GSSAPI
21:09:32.281 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
21:09:32.281 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=150), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.79.62:39595, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.79.61:40625, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.15.79.60:43573, [142ms] Sub RPC ConnectToMaster: received response from server master-127.15.79.61:40625: OK, deferred=null) after lookup
21:09:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.284688 16275 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.287773 16275 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Initialized, Role: FOLLOWER
21:09:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.288791 16275 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.291544 16284 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.293426 16275 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: configured and running, proceeding with master startup.
21:09:32.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.293464 16284 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:32.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:32.327 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x485cbf4d, L:/127.0.0.1:52106 - R:/127.15.79.60:43573] using SASL/GSSAPI
21:09:32.347 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.15.79.60:43573: Catalog manager is not initialized. State: Starting
21:09:32.351 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf073e06b, L:/127.0.0.1:36172 - R:/127.15.79.62:39595] using SASL/GSSAPI
21:09:32.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.365502 16295 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:32.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.365757 16295 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: loading cluster ID for follower catalog manager: success
21:09:32.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:32 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543371, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.374141 16295 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: acquiring CA information for follower catalog manager: success
21:09:32.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.379554 16295 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:32.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.490417 16250 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 1 FOLLOWER]: Advancing to term 2
21:09:32.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.522249 16284 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:32.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.522986 16284 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:32.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.524770 16302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:32.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:32.525557 16302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.032521 16213 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:33.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.034255 16213 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:33.287 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.62:39595
21:09:33.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:09:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.287664 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16003
21:09:33.320 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.79.62:39595(127.15.79.62:39595)] handling channelInactive
21:09:33.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.79.62:39595(127.15.79.62:39595)] cleaning up while in state READY due to: connection closed
21:09:33.322 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:33.323 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.61:40625
21:09:33.323 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:09:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.323509 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16076
21:09:33.346 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.79.61:40625(127.15.79.61:40625)] handling channelInactive
21:09:33.347 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.79.61:40625(127.15.79.61:40625)] cleaning up while in state READY due to: connection closed
21:09:33.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:33.350 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.60:43573
21:09:33.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:09:33.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.355932 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16170
21:09:33.375 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.79.60:43573(127.15.79.60:43573)] handling channelInactive
21:09:33.375 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:33.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.79.60:43573(127.15.79.60:43573)] cleaning up while in state READY due to: connection closed
21:09:33.375 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.62:39595
21:09:33.376 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.377398 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:33.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:33.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:33.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.671406 16307 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:33.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.671952 16307 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:33.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.672582 16307 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:33.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.703728 16307 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.703945 16307 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.704216 16307 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.704421 16307 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.704654 16307 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:33.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.704851 16307 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:33.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.705050 16307 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.740710 16307 master_runner.cc:386] Master server non-default flags:
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.741938 16307 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.742934 16307 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.757870 16314 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:34.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:34.850556 15106 debug-util.cc:398] Leaking SignalData structure 0x7b08000aec80 after lost signal to thread 15041
21:09:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:34.851045 15106 debug-util.cc:398] Leaking SignalData structure 0x7b08000996a0 after lost signal to thread 15109
21:09:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.758312 16316 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:33.759356 16307 server_base.cc:1034] running on GCE node
21:09:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:33.757880 16313 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:34.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.875775 16307 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:34.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.878599 16307 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.880024 16307 hybrid_clock.cc:648] HybridClock initialized: now 1736543374879937 us; error 96 us; skew 500 ppm
21:09:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.880880 16307 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:34.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:34 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543374, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:34.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.886535 16307 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.891717 16307 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.892639 16307 fs_manager.cc:362] Metadata directory not provided
21:09:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.892844 16307 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:34.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.900306 16307 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.003s
21:09:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.904423 16325 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:34.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.905449 16307 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:09:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.905716 16307 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:34.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.907892 16307 fs_report.cc:389] FS layout report
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:34.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:34.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.954442 16307 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.961158 16307 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:34.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:34.961562 16307 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:35.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.027712 16307 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:35.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.027860 16376 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:35.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.030418 16307 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:35.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.032543 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16307
21:09:35.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:35.033 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.61:40625
21:09:35.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.034857 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:35.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:35.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.041536 16377 sys_catalog.cc:263] Verifying existing consensus state
21:09:35.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.046684 16377 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:35.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.081743 16377 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.097266 16377 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:35.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.098006 16377 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap complete.
21:09:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.124423 16377 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.127854 16377 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:35.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.128821 16377 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:35.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.131776 16381 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:35.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.132644 16381 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:35.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.133095 16377 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:35.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.160547 16392 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:35.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.160822 16392 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: success
21:09:35.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.168190 16392 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: acquiring CA information for follower catalog manager: success
21:09:35.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.178109 16392 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:35.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:35 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543374, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.214972 16307 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111),
21:09:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.244670 16307 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.374034 16379 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.374697 16379 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.375443 16379 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.406549 16379 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.406803 16379 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.407183 16379 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.407457 16379 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.407717 16379 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.407999 16379 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.408247 16379 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.444608 16379 master_runner.cc:386] Master server non-default flags:
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:35.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:35.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.445911 16379 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:35.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:35.446882 16379 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:35.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.462882 16402 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:36.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:36.502879 16410 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:36.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:36.503505 16410 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:36.521921 16328 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:36.537900 16328 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625): Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:36.543241 16326 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:36.543674 16326 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:36.550901 16410 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:36.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:36.551429 16410 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
21:09:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:36.859009 16400 debug-util.cc:398] Leaking SignalData structure 0x7b080001cac0 after lost signal to thread 16379
21:09:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.144666 16400 kernel_stack_watchdog.cc:198] Thread 16379 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:636 for 394ms:
21:09:37.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.145174 16379 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.682s user 0.583s sys 0.970s
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.145514 16379 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.682s user 0.583s sys 0.970s
21:09:37.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:35.462961 16401 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.146435 16404 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:37.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.150489 16403 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1673 milliseconds
21:09:37.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.150532 16379 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:37.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.152040 16379 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:37.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.155036 16379 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.156518 16379 hybrid_clock.cc:648] HybridClock initialized: now 1736543377156410 us; error 76 us; skew 500 ppm
21:09:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.157670 16379 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:37.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:37 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543377, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:37.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.164984 16379 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.172194 16379 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:37.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.173277 16379 fs_manager.cc:362] Metadata directory not provided
21:09:37.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.173549 16379 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.181576 16379 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
21:09:37.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.186070 16419 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:37.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.187059 16379 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
21:09:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.187335 16379 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.189572 16379 fs_report.cc:389] FS layout report
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:37.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:37.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.248788 16379 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.254472 16379 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.254863 16379 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:37.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.336618 16379 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.336692 16471 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:37.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.339504 16379 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:37.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.347981 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16379
21:09:37.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:37.348 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.60:43573
21:09:37.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:09:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.350481 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.350872 16472 sys_catalog.cc:263] Verifying existing consensus state
21:09:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.356707 16472 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.405890 16472 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.425952 16472 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.427577 16472 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap complete.
21:09:37.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.461556 16472 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:37.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.464923 16472 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:37.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.466050 16472 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.477473 16476 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.478297 16476 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.479205 16472 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.525190 16487 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:37.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.525446 16487 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:37.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.541455 16487 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.546607 16487 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:37 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543377, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.600557 16379 master_runner.cc:186] Error getting master registration for 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111),
21:09:37.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.629488 16379 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.752163 16474 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.752710 16474 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.753296 16474 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.784356 16474 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.784606 16474 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.784924 16474 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.785141 16474 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.785331 16474 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.785516 16474 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.785657 16474 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.821171 16474 master_runner.cc:386] Master server non-default flags:
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:37.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:37.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.822511 16474 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:37.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:37.823741 16474 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:37.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.844106 16499 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:38.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.327139 16505 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.327968 16505 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:38.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.343779 16505 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:38.369573 16326 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:38.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.385226 16447 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ad20382b782c44a3a984136f62be401f" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" is_pre_election: true
21:09:38.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.386229 16447 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ad20382b782c44a3a984136f62be401f in term 2.
21:09:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.387355 16328 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.388438 16505 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Leader pre-election won for term 3
21:09:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.388793 16505 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.389099 16505 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 2 FOLLOWER]: Advancing to term 3
21:09:38.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.398434 16505 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.410295 16505 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 election: Requested vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.410562 16447 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ad20382b782c44a3a984136f62be401f" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.411162 16447 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 2 FOLLOWER]: Advancing to term 3
21:09:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:38.414593 16326 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:38.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.424918 16447 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ad20382b782c44a3a984136f62be401f in term 3.
21:09:38.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.425932 16328 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:38.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.426671 16505 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Leader election won for term 3
21:09:38.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.428586 16505 raft_consensus.cc:695] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 LEADER]: Becoming Leader. State: Replica: ad20382b782c44a3a984136f62be401f, State: Running, Role: LEADER
21:09:38.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.429513 16505 consensus_queue.cc:237] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:38.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.444875 16510 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ad20382b782c44a3a984136f62be401f. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.445828 16510 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: LEADER
21:09:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.461386 16512 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:38.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.478780 16512 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:38.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.487084 16512 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:38.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.493454 16512 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:38.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.498517 16512 catalog_manager.cc:1505] Loading token signing keys...
21:09:38.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.502820 16512 catalog_manager.cc:5910] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Loaded TSK: 0
21:09:38.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.504155 16512 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:38.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.930277 16447 raft_consensus.cc:1270] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Refusing update from remote peer ad20382b782c44a3a984136f62be401f: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
21:09:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.934811 16510 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [LEADER]: Connected to new peer: Peer: permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
21:09:38.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.977172 16513 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ad20382b782c44a3a984136f62be401f. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:38.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:38.978016 16513 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.017335 16515 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.019454 16515 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: LEADER
21:09:39.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.048045 16513 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.048767 16513 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:39.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:39.057125 16326 consensus_peers.cc:487] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f -> Peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Couldn't send request to peer 6e646947c6524250b8154c630cc52ebb. Status: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:09:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.848995 16498 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:37.850857 16501 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:39.339152 16500 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1497 milliseconds
21:09:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.339207 16474 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.345865 16474 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:39.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.349778 16474 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.351244 16474 hybrid_clock.cc:648] HybridClock initialized: now 1736543379351172 us; error 72 us; skew 500 ppm
21:09:39.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.352407 16474 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:39.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.366195 16474 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:39.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.373219 16474 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:39.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.374616 16474 fs_manager.cc:362] Metadata directory not provided
21:09:39.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.374997 16474 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:39.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.385798 16474 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
21:09:39.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.390156 16528 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:39.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.391140 16474 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
21:09:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.391412 16474 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:39.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.393829 16474 fs_report.cc:389] FS layout report
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:39.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.443360 16474 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.448951 16474 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.449343 16474 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.533049 16474 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:39.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.533161 16579 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.535867 16474 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:39.539 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:39.540 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.61:40625(127.15.79.61:40625)] connecting to peer
21:09:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.538360 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16474
21:09:39.542 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.79.61:40625(127.15.79.61:40625)] Successfully connected to peer
21:09:39.545 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:39.545 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:39.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:39.550 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:39.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.550639 16580 sys_catalog.cc:263] Verifying existing consensus state
21:09:39.550 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:39.552 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.61
21:09:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.557332 16580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:39.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543364, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:39.579 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x72b5392e, L:/127.0.0.1:36550 - R:/127.15.79.61:40625] using SASL/GSSAPI
21:09:39.583 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:09:39.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:09:39.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.618069 16580 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:39.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543374, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.60@KRBTEST.COM
21:09:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.656044 16580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:39.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.657023 16580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap complete.
21:09:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:09:39.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:39.685524 15676 mini_kdc.cc:317] Time spent kinit for test-admin: real 0.102s user 0.001s sys 0.011s
21:09:39.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:39.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.694797 16580 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:39.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.697111 16580 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Initialized, Role: FOLLOWER
21:09:39.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.698005 16580 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.700691 16588 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.701962 16588 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:39.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.702536 16580 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: configured and running, proceeding with master startup.
21:09:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.703171 16555 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 2 FOLLOWER]: Advancing to term 3
21:09:39.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.729504 16600 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:39.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.730805 16600 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:39.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.733920 16588 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: New leader ad20382b782c44a3a984136f62be401f. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:39.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.734450 16588 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:39.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.754977 16602 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:39.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.755242 16602 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: loading cluster ID for follower catalog manager: success
21:09:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.762868 16602 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: acquiring CA information for follower catalog manager: success
21:09:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:39 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:39.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:39.769114 16602 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.147408 16516 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.148495 16516 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: LEADER
21:09:40.686 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.62:39595
21:09:40.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:09:40.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.686892 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16307
21:09:40.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:40.710 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.61:40625
21:09:40.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:09:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.710675 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16379
21:09:40.733 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.79.61:40625(127.15.79.61:40625)] handling channelInactive
21:09:40.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:40.733 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.79.61:40625(127.15.79.61:40625)] cleaning up while in state READY due to: connection closed
21:09:40.733 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.60:43573
21:09:40.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:09:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.734293 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16474
21:09:40.756 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:40.756 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.62:39595
21:09:40.756 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:40.758117 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:41.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.046231 16609 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.046808 16609 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.047706 16609 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.078306 16609 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.078536 16609 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.078889 16609 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.079154 16609 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.079401 16609 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.079644 16609 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:41.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.079836 16609 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:41.114804 16609 master_runner.cc:386] Master server non-default flags:
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:41.116371 16609 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:41.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:41.117396 16609 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:41.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.130811 16616 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:42.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:41.130820 16615 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:42.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.244192 16617 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:09:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.245826 16618 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:42.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.245841 16609 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:42.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.249243 16609 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:42.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.252084 16609 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:42.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.253504 16609 hybrid_clock.cc:648] HybridClock initialized: now 1736543382253439 us; error 53 us; skew 500 ppm
21:09:42.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.254318 16609 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:42.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:42 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543382, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:42.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.259954 16609 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.264866 16609 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:42.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.265754 16609 fs_manager.cc:362] Metadata directory not provided
21:09:42.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.265967 16609 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.273765 16609 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
21:09:42.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.277796 16627 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:42.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.278744 16609 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.279047 16609 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.281311 16609 fs_report.cc:389] FS layout report
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:42.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.340318 16609 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:42.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.345682 16609 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:42.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.346145 16609 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.415791 16609 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.415858 16678 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:42.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.418521 16609 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:42.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:42.421 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.61:40625
21:09:42.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:09:42.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.421052 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16609
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.423118 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:42.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:42.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:42.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:42.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.428225 16679 sys_catalog.cc:263] Verifying existing consensus state
21:09:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.433094 16679 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:42.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.474645 16679 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.495050 16679 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.495723 16679 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap complete.
21:09:42.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.518548 16679 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:42.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.520787 16679 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.521507 16679 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.524025 16683 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.524875 16683 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:42.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.525717 16679 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.555687 16694 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:42.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.555953 16694 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: success
21:09:42.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.563517 16694 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: acquiring CA information for follower catalog manager: success
21:09:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.568862 16694 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.581239 16609 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111),
21:09:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.610723 16609 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:42.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.762377 16681 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.762969 16681 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:42.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.763713 16681 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.796162 16681 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.796411 16681 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:42.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.796801 16681 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.797062 16681 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.797325 16681 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.797573 16681 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.797767 16681 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.833818 16681 master_runner.cc:386] Master server non-default flags:
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:42.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.835381 16681 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:42.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.836675 16681 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.853616 16703 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:43.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:43.839006 16709 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:43.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:43.839603 16709 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:43.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:43.842602 16709 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:43.858240 16630 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:43.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:43.878371 16628 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:43.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:43.881052 16630 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625): Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:43.881593 16630 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:43.882452 16709 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
21:09:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.854008 16704 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:42.855686 16681 server_base.cc:1034] running on GCE node
21:09:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:42.855370 16706 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:44.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.004040 16681 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:44.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.008587 16681 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:44.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.009985 16681 hybrid_clock.cc:648] HybridClock initialized: now 1736543384009939 us; error 41 us; skew 500 ppm
21:09:44.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.010795 16681 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:44 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543384, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:44.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.016759 16681 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:44.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.022001 16681 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:44.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.022905 16681 fs_manager.cc:362] Metadata directory not provided
21:09:44.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.023113 16681 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.030854 16681 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
21:09:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.035360 16718 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:44.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.036522 16681 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:09:44.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.036832 16681 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.038949 16681 fs_report.cc:389] FS layout report
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:44.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:44.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.094789 16681 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:44.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.101794 16681 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:44.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.102375 16681 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.188910 16681 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.189029 16770 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:44.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.191666 16681 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:44.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.196444 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16681
21:09:44.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:44.197 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.60:43573
21:09:44.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.198909 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:44.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:44.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.203179 16771 sys_catalog.cc:263] Verifying existing consensus state
21:09:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.210151 16771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:44.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.259816 16771 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.281258 16771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:44.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.282168 16771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap complete.
21:09:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.315883 16771 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.319259 16771 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:44.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.320360 16771 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.326109 16775 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:44.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.327350 16775 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.327823 16771 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:44.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.367214 16786 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:44.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.367487 16786 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:44.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.381692 16786 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.386613 16786 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:44 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543384, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.461035 16681 master_runner.cc:186] Error getting master registration for 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111),
21:09:44.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.493784 16681 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.616252 16773 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.616987 16773 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.617969 16773 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.652932 16773 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.653188 16773 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.653577 16773 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.653847 16773 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.654109 16773 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.654363 16773 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.654557 16773 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.690652 16773 master_runner.cc:386] Master server non-default flags:
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:44.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.692234 16773 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.693327 16773 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.708913 16798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:45.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.503099 16803 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:45.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.503760 16803 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.507331 16803 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.508375 16654 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f" is_pre_election: true
21:09:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.509231 16654 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 3.
21:09:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.510767 16722 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters:
21:09:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.512007 16803 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Leader pre-election won for term 4
21:09:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.512574 16803 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.514263 16803 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 3 FOLLOWER]: Advancing to term 4
21:09:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:45.525475 16719 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:45.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.523092 16803 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:45.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.533031 16654 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f"
21:09:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.533591 16654 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 3 FOLLOWER]: Advancing to term 4
21:09:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.543212 16654 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 4.
21:09:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.544430 16722 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters:
21:09:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.546638 16808 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Leader election won for term 4
21:09:45.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:45.549299 16719 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:45.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.550671 16803 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 election: Requested vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:45.554936 16719 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.560909 16808 raft_consensus.cc:695] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 LEADER]: Becoming Leader. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Running, Role: LEADER
21:09:45.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.561945 16808 consensus_queue.cc:237] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.573462 16803 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.574278 16803 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.577029 16811 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:45.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.586056 16811 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:45.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.590633 16811 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:45.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.590874 16811 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:45.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.606415 16811 catalog_manager.cc:1505] Loading token signing keys...
21:09:45.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.622483 16811 catalog_manager.cc:5910] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Loaded TSK: 0
21:09:45.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.623754 16811 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:45.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.709990 16800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:45.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:44.710182 16797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:45.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:44.711009 16773 server_base.cc:1034] running on GCE node
21:09:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.876785 16773 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.879854 16773 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:45.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.881378 16773 hybrid_clock.cc:648] HybridClock initialized: now 1736543385881310 us; error 73 us; skew 500 ppm
21:09:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.882472 16773 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:45.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:45 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543385, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:45.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.889626 16773 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:45.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.895359 16773 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.896560 16773 fs_manager.cc:362] Metadata directory not provided
21:09:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.896893 16773 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.904768 16773 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
21:09:45.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.909173 16818 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:45.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.910161 16773 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
21:09:45.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.910482 16773 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:45.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:45.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.912670 16773 fs_report.cc:389] FS layout report
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:45.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:45.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.982952 16773 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.988422 16773 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:45.988911 16773 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:46.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:46.003950 16719 consensus_peers.cc:487] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 -> Peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Couldn't send request to peer 6e646947c6524250b8154c630cc52ebb. Status: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:09:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.070324 16773 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:46.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.070730 16869 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:46.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.073132 16773 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:46.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.079391 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16773
21:09:46.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:46.080 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.61:40625(127.15.79.61:40625)] connecting to peer
21:09:46.082 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.79.61:40625(127.15.79.61:40625)] Successfully connected to peer
21:09:46.086 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:46.086 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:46.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:46 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.089057 16870 sys_catalog.cc:263] Verifying existing consensus state
21:09:46.089 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:46.089 [INFO - kudu-nio-0] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:46.090 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.61
21:09:46.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.095955 16870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:46.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:46 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:46.127 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb72a8165, L:/127.0.0.1:36600 - R:/127.15.79.61:40625] using SASL/GSSAPI
21:09:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.149701 16870 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.164534 16654 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Refusing update from remote peer b42c9a70f6c1438385ef2f0d250f8fc8: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
21:09:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.166352 16870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:46.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.166288 16803 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
21:09:46.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.167239 16870 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap complete.
21:09:46.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.176085 16875 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.178805 16875 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.191857 16875 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.192752 16875 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.201323 16803 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.202072 16803 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.211661 16870 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:46.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.215034 16870 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Initialized, Role: FOLLOWER
21:09:46.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.215956 16870 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:46.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.218720 16881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.220376 16881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:46.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.220443 16870 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: configured and running, proceeding with master startup.
21:09:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.266295 16892 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.266546 16892 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: loading cluster ID for follower catalog manager: success
21:09:46.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:46 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543385, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:46.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.273990 16892 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: acquiring CA information for follower catalog manager: success
21:09:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.279503 16892 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.416258 16845 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 3 FOLLOWER]: Advancing to term 4
21:09:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.430198 16899 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.431409 16899 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.435006 16881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.435505 16881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:46.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.876582 16803 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:46.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:46.879751 16803 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:47.131 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.62:39595
21:09:47.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:09:47.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.131516 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16609
21:09:47.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:47.154 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.61:40625
21:09:47.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:09:47.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.155414 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16681
21:09:47.188 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.79.61:40625(127.15.79.61:40625)] handling channelInactive
21:09:47.188 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.79.61:40625(127.15.79.61:40625)] cleaning up while in state READY due to: connection closed
21:09:47.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:47.188 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.60:43573
21:09:47.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:09:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.189246 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16773
21:09:47.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:47.210 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.62:39595
21:09:47.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:09:47.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.212118 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:47.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.508494 16904 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:47.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.509083 16904 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:47.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.509737 16904 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.541277 16904 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:47.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.541527 16904 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.541889 16904 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.542161 16904 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.542421 16904 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.542671 16904 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:47.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.542867 16904 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:47.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.578538 16904 master_runner.cc:386] Master server non-default flags:
21:09:47.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:47.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:47.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.580089 16904 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:47.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:47.581250 16904 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:47.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.596329 16911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:48.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:47.597278 16910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:48.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:48.717921 16912 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:09:48.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:48.719758 16913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:48.722180 16904 thread.cc:635] OpenStack (cloud detector) Time spent creating pthread: real 1.129s user 0.457s sys 0.665s
21:09:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:48.722512 16904 thread.cc:601] OpenStack (cloud detector) Time spent starting thread: real 1.129s user 0.457s sys 0.665s
21:09:48.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.722779 16904 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.724176 16904 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:48.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.727329 16904 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.728771 16904 hybrid_clock.cc:648] HybridClock initialized: now 1736543388728714 us; error 40 us; skew 500 ppm
21:09:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.729848 16904 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:48.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:48 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543388, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:48.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.738190 16904 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:48.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.743983 16904 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.745225 16904 fs_manager.cc:362] Metadata directory not provided
21:09:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.745489 16904 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:48.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.756129 16904 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
21:09:48.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.761564 16923 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:48.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.762739 16904 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:09:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.763123 16904 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.766373 16904 fs_report.cc:389] FS layout report
21:09:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:48.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:48.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.849470 16904 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:48.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.854605 16904 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.855059 16904 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:48.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.924844 16904 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:48.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.924938 16974 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:48.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.927614 16904 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:48.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.935428 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16904
21:09:48.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:48.936 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.61:40625
21:09:48.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:09:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.937619 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:48.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:48.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.938621 16975 sys_catalog.cc:263] Verifying existing consensus state
21:09:48.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.943893 16975 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:48.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.979490 16975 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:48.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.996318 16975 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:48.997105 16975 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap complete.
21:09:49.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.019899 16975 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:49.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.022202 16975 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:49.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.023027 16975 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.025779 16979 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.026603 16979 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.027591 16975 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:49.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.066387 16990 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:49.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.066819 16990 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: success
21:09:49.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:49 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543388, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.073817 16990 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: acquiring CA information for follower catalog manager: success
21:09:49.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.079437 16990 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:49.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.087775 16904 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111),
21:09:49.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.106156 16904 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:49.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.268236 16977 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.268829 16977 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:49.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.269527 16977 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.301095 16977 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.301398 16977 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:49.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.301697 16977 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.301939 16977 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:49.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.302178 16977 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.302373 16977 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.302508 16977 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.340428 16977 master_runner.cc:386] Master server non-default flags:
21:09:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:49.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.342120 16977 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.343473 16977 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.360348 17001 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:50.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.361991 17000 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:49.362521 17003 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:49.361832 16977 server_base.cc:1034] running on GCE node
21:09:50.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.493728 16977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.497328 16977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.498762 16977 hybrid_clock.cc:648] HybridClock initialized: now 1736543390498677 us; error 100 us; skew 500 ppm
21:09:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.499783 16977 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:50.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:50 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543390, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:50.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.511397 16977 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.516994 16977 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.518137 16977 fs_manager.cc:362] Metadata directory not provided
21:09:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.518368 16977 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:50.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.526589 16977 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
21:09:50.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.531355 17012 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:50.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.532447 16977 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.532821 16977 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.535058 16977 fs_report.cc:389] FS layout report
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:50.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:50.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.598644 16977 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:50.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.606504 16977 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:50.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.607102 16977 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.630728 17020 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:50.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.631489 17020 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:50.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.635010 17020 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:50.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:50.637957 16926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:50.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:50.646703 16926 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625): Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:50.648288 16924 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.648725 16924 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:50.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.649907 17020 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
21:09:50.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.702258 16977 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:50.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.702512 17067 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.705332 16977 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:50.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.712743 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 16977
21:09:50.713 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:50.713 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.60:43573
21:09:50.713 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.715101 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:50.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.718830 17068 sys_catalog.cc:263] Verifying existing consensus state
21:09:50.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.726460 17068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:50.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.778730 17068 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.801232 17068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.802237 17068 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap complete.
21:09:50.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.836884 17068 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:50.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.840270 17068 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.841342 17068 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.844280 17072 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:50.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.845341 17072 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.845955 17068 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:50.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:50.909758 17083 catalog_manager.cc:796] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:09:50.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:50 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543390, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:50 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543390, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.002926 16977 master_runner.cc:186] Error getting master registration for 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111),
21:09:51.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.026489 16977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:51.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.079553 17070 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:51.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.080129 17070 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:51.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.080817 17070 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.111338 17070 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.111670 17070 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.112092 17070 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.112396 17070 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.112715 17070 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:51.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.112990 17070 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:51.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.113197 17070 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.149704 17070 master_runner.cc:386] Master server non-default flags:
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:51.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:51.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.151298 17070 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.152624 17070 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:51.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.166143 17092 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:51.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.917584 17083 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:51.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.917856 17083 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.924970 17083 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:51.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.930478 17083 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:52.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.215981 17101 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:52.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.216637 17101 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:52.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.219882 17101 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.220882 16950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f" is_pre_election: true
21:09:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.221647 16950 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 4.
21:09:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.222867 17015 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters:
21:09:52.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.224371 17101 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Leader pre-election won for term 5
21:09:52.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:52.225490 17013 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.224900 17101 raft_consensus.cc:491] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:52.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.226205 17101 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 4 FOLLOWER]: Advancing to term 5
21:09:52.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.235209 17101 raft_consensus.cc:513] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.238471 16950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "ad20382b782c44a3a984136f62be401f"
21:09:52.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.239009 16950 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 4 FOLLOWER]: Advancing to term 5
21:09:52.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.242416 17101 leader_election.cc:290] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 election: Requested vote from peers ad20382b782c44a3a984136f62be401f (127.15.79.62:39595), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:52.242937 17013 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:52.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.248222 16950 raft_consensus.cc:2463] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b42c9a70f6c1438385ef2f0d250f8fc8 in term 5.
21:09:52.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:52.249228 17013 leader_election.cc:336] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.249879 17015 leader_election.cc:304] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:52.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.250604 17101 raft_consensus.cc:2798] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Leader election won for term 5
21:09:52.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.252663 17101 raft_consensus.cc:695] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 LEADER]: Becoming Leader. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Running, Role: LEADER
21:09:52.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.253542 17101 consensus_queue.cc:237] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.263931 17104 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.264740 17104 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:52.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.267398 17106 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:52.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.277328 17106 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:52.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.281733 17106 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:52.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.281972 17106 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:52.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.286861 17106 catalog_manager.cc:1505] Loading token signing keys...
21:09:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.291615 17106 catalog_manager.cc:5910] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Loaded TSK: 0
21:09:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.292943 17106 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:52.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:51.168249 17070 server_base.cc:1034] running on GCE node
21:09:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.169003 17095 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:51.166926 17093 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.298213 17070 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.301147 17070 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.302585 17070 hybrid_clock.cc:648] HybridClock initialized: now 1736543392302521 us; error 70 us; skew 500 ppm
21:09:52.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.303364 17070 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:52 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543392, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.309269 17070 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:52.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.314435 17070 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.315255 17070 fs_manager.cc:362] Metadata directory not provided
21:09:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.315433 17070 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:52.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.323431 17070 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
21:09:52.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.327833 17112 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:52.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.328848 17070 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.329126 17070 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.331295 17070 fs_report.cc:389] FS layout report
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:52.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:52.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:52.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.399271 17070 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.404878 17070 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.405272 17070 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:52.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.496165 17070 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:52.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.496603 17163 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:52.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.499704 17070 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.502339 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 17070
21:09:52.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:52.503 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.61:40625(127.15.79.61:40625)] connecting to peer
21:09:52.505 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.15.79.61:40625(127.15.79.61:40625)] Successfully connected to peer
21:09:52.508 [DEBUG - kudu-nio-1] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:52.509 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:52 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:52.512 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:09:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.512791 17164 sys_catalog.cc:263] Verifying existing consensus state
21:09:52.517 [INFO - kudu-nio-1] (SecurityContext.java:259) Successfully refreshed Kerberos credentials from ticket cache
21:09:52.518 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.15.79.61
21:09:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.520298 17164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:52.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:52 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.15.79.61@KRBTEST.COM
21:09:52.551 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x491eb4ae, L:/127.0.0.1:47966 - R:/127.15.79.61:40625] using SASL/GSSAPI
21:09:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.571813 17164 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.589805 17164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.591074 17164 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap complete.
21:09:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.620615 17164 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.623106 17164 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e646947c6524250b8154c630cc52ebb, State: Initialized, Role: FOLLOWER
21:09:52.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.624007 17164 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.626602 17168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.627424 17168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.628263 17164 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: configured and running, proceeding with master startup.
21:09:52.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.666095 16950 raft_consensus.cc:1270] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Refusing update from remote peer b42c9a70f6c1438385ef2f0d250f8fc8: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
21:09:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.667850 17104 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
21:09:52.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.675652 17179 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:52.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.675944 17179 catalog_manager.cc:1553] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: loading cluster ID for follower catalog manager: success
21:09:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:52 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543392, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:52.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.689002 17179 catalog_manager.cc:1575] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: acquiring CA information for follower catalog manager: success
21:09:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.706861 17183 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.707702 17183 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.712270 17179 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:52.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.725749 17183 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.726390 17183 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:52.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.731127 17104 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.731819 17104 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:52.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.758937 17139 raft_consensus.cc:3054] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 4 FOLLOWER]: Advancing to term 5
21:09:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.768992 17139 raft_consensus.cc:1270] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [term 5 FOLLOWER]: Refusing update from remote peer b42c9a70f6c1438385ef2f0d250f8fc8: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
21:09:52.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.770720 17104 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
21:09:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.789451 17191 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.790596 17191 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.794703 17168 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: SysCatalogTable state changed. Reason: New leader b42c9a70f6c1438385ef2f0d250f8fc8. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:52.795400 17168 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb [sys.catalog]: This master's current role is: FOLLOWER
21:09:53.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.159808 17104 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:53.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.160607 17104 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: LEADER
21:09:53.557 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.62:39595
21:09:53.557 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:09:53.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.557691 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16904
21:09:53.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:53.595 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.61:40625
21:09:53.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:09:53.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.596010 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 16977
21:09:53.624 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.15.79.61:40625(127.15.79.61:40625)] handling channelInactive
21:09:53.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.15.79.61:40625(127.15.79.61:40625)] cleaning up while in state READY due to: connection closed
21:09:53.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:53.625 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.15.79.60:43573
21:09:53.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:09:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.625911 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 17070
21:09:53.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:53.647 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.62:39595
21:09:53.648 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:53.649585 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.944829 17193 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.945343 17193 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.945909 17193 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.976495 17193 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.976719 17193 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:53.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.976979 17193 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:53.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.977269 17193 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:53.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.977576 17193 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.977914 17193 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:53.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:53.978163 17193 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:54.013903 17193 master_runner.cc:386] Master server non-default flags:
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.62.keytab
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.62
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.62:39595
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.62
21:09:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42193
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/logs
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:54.015473 17193 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:54.016623 17193 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:54.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:54.030365 17200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:54.032830 17193 server_base.cc:1034] running on GCE node
21:09:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:54.031744 17202 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:54.030757 17199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.138137 17193 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:55.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.140609 17193 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.141934 17193 hybrid_clock.cc:648] HybridClock initialized: now 1736543395141895 us; error 57 us; skew 500 ppm
21:09:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.142724 17193 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:55.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:55 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543395, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:55.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.148265 17193 init.cc:377] Logged in from keytab as kudu/127.15.79.62@KRBTEST.COM (short username kudu)
21:09:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.153789 17193 webserver.cc:458] Webserver started at http://127.15.79.62:42193/ using document root <none> and password file <none>
21:09:55.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.154750 17193 fs_manager.cc:362] Metadata directory not provided
21:09:55.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.154963 17193 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:55.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.162585 17193 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
21:09:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.166630 17211 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.167577 17193 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.167891 17193 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad20382b782c44a3a984136f62be401f"
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:21 on dist-test-slave-4hht"
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.170140 17193 fs_report.cc:389] FS layout report
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/wal
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/data
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:55.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.213966 17193 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.219188 17193 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.219658 17193 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:55.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.288203 17193 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.62:39595
21:09:55.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.288339 17262 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.62:39595 every 8 connection(s)
21:09:55.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.290877 17193 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-0/data/info.pb
21:09:55.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.293540 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 17193
21:09:55.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:55.294 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.61:40625
21:09:55.294 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:09:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.295622 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:55.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:55.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:55.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.301311 17263 sys_catalog.cc:263] Verifying existing consensus state
21:09:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.306595 17263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap starting.
21:09:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.341591 17263 log.cc:826] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Log is configured to *not* fsync() on all Append() calls
21:09:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.359323 17263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.360039 17263 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Bootstrap complete.
21:09:55.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.384392 17263 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.386821 17263 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad20382b782c44a3a984136f62be401f, State: Initialized, Role: FOLLOWER
21:09:55.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.387715 17263 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.390429 17267 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 5 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:55.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.391279 17267 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: FOLLOWER
21:09:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.392168 17263 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: configured and running, proceeding with master startup.
21:09:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.423373 17278 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:55.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.423630 17278 catalog_manager.cc:1553] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: loading cluster ID for follower catalog manager: success
21:09:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:55 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543395, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.62@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.431861 17278 catalog_manager.cc:1575] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: acquiring CA information for follower catalog manager: success
21:09:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.438184 17278 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.453029 17193 master_runner.cc:186] Error getting master registration for 127.15.79.61:40625: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111),
21:09:55.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.482331 17193 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:55.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.629281 17265 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.629882 17265 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:55.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.630580 17265 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.662933 17265 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.663226 17265 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.663617 17265 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.663887 17265 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.664149 17265 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.664408 17265 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.664650 17265 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.701537 17265 master_runner.cc:386] Master server non-default flags:
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.61.keytab
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.61
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.61:40625
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.61
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39379
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/logs
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:55.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.703054 17265 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:55.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.704202 17265 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:55.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.719429 17288 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.843963 17296 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.844614 17296 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:56.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.721009 17290 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:55.722534 17265 server_base.cc:1034] running on GCE node
21:09:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:55.719668 17287 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:56.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:56.850503 17214 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:56.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.852525 17265 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:56.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.855391 17265 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.856812 17265 hybrid_clock.cc:648] HybridClock initialized: now 1736543396856742 us; error 67 us; skew 500 ppm
21:09:56.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.856803 17296 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:56.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.857863 17265 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:56.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:56.861191 17214 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625): Network error: Client connection negotiation failed: client connection to 127.15.79.61:40625: connect: Connection refused (error 111)
21:09:56.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:56.862882 17212 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:56.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.863189 17212 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ad20382b782c44a3a984136f62be401f; no voters: 6e646947c6524250b8154c630cc52ebb, b42c9a70f6c1438385ef2f0d250f8fc8
21:09:56.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.864184 17296 raft_consensus.cc:2743] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Leader pre-election lost for term 6. Reason: could not achieve majority
21:09:56.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:56 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543396, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:56.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.869362 17265 init.cc:377] Logged in from keytab as kudu/127.15.79.61@KRBTEST.COM (short username kudu)
21:09:56.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.877421 17265 webserver.cc:458] Webserver started at http://127.15.79.61:39379/ using document root <none> and password file <none>
21:09:56.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.878330 17265 fs_manager.cc:362] Metadata directory not provided
21:09:56.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.878546 17265 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.886974 17265 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
21:09:56.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.891677 17305 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.892939 17265 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
21:09:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.893340 17265 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:22 on dist-test-slave-4hht"
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.895699 17265 fs_report.cc:389] FS layout report
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/wal
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/data
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:56.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:56.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:56.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.944944 17265 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:56.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.950639 17265 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:56.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:56.951136 17265 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:57.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.029078 17265 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.61:40625
21:09:57.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.029192 17356 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.61:40625 every 8 connection(s)
21:09:57.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.032032 17265 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-1/data/info.pb
21:09:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.040009 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 17265
21:09:57.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:57.040 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.15.79.60:43573
21:09:57.040 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:09:57.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.042450 15676 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:57.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu
21:09:57.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:57.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:09:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:09:57.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.043812 17357 sys_catalog.cc:263] Verifying existing consensus state
21:09:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.050171 17357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap starting.
21:09:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.090880 17357 log.cc:826] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Log is configured to *not* fsync() on all Append() calls
21:09:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.110110 17357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.110842 17357 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: Bootstrap complete.
21:09:57.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.141449 17357 raft_consensus.cc:357] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:57.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.143710 17357 raft_consensus.cc:738] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Becoming Follower/Learner. State: Replica: b42c9a70f6c1438385ef2f0d250f8fc8, State: Initialized, Role: FOLLOWER
21:09:57.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.144567 17357 consensus_queue.cc:260] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.147207 17361 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 5 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.148208 17361 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:57.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.149595 17357 sys_catalog.cc:564] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: configured and running, proceeding with master startup.
21:09:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.201586 17374 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.201933 17374 catalog_manager.cc:1553] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: loading cluster ID for follower catalog manager: success
21:09:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.220918 17374 catalog_manager.cc:1575] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: acquiring CA information for follower catalog manager: success
21:09:57.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:57 dist-test-slave-4hht krb5kdc[15687](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543396, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.61@KRBTEST.COM for kudu/127.15.79.62@KRBTEST.COM
21:09:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.226940 17374 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:09:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.268570 17265 master_runner.cc:186] Error getting master registration for 127.15.79.60:43573: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111),
21:09:57.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.321061 17265 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:09:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.456833 17359 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:09:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.457373 17359 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:09:57.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.458197 17359 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:09:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.506594 17359 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:09:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.506809 17359 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:09:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.507112 17359 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:09:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.507323 17359 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:09:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.507514 17359 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:09:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.507759 17359 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:09:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.507990 17359 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38817/jwks.json
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.545378 17359 master_runner.cc:386] Master server non-default flags:
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.79.20:38497
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:09:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.79.62:39595,127.15.79.61:40625,127.15.79.60:43573
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/kudu_127.15.79.60.keytab
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.15.79.60
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.79.60:43573
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38817/jwks.json
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.79.60
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42079
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/logs
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bda246888fb83d6b8131ddbb04f062f3e03180a8
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 10 Jan 2025 20:43:23 UTC on a633016ff5c9
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4116
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:09:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.546545 17359 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:09:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:57.547472 17359 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:09:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.562979 17385 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:58.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.406730 17392 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:09:58.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.407398 17392 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:58.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.438984 17392 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: Requested pre-vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:58.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:58.503361 17212 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.510888 17332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ad20382b782c44a3a984136f62be401f" candidate_term: 6 candidate_status { last_received { term: 5 index: 8 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" is_pre_election: true
21:09:58.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.512008 17332 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ad20382b782c44a3a984136f62be401f in term 5.
21:09:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.514490 17214 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.515336 17392 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Leader pre-election won for term 6
21:09:58.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.515689 17392 raft_consensus.cc:491] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:09:58.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.516003 17392 raft_consensus.cc:3054] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 5 FOLLOWER]: Advancing to term 6
21:09:58.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.532924 17392 raft_consensus.cc:513] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 6 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:58.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.538376 17332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ad20382b782c44a3a984136f62be401f" candidate_term: 6 candidate_status { last_received { term: 5 index: 8 } } ignore_live_leader: false dest_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8"
21:09:58.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.538996 17332 raft_consensus.cc:3054] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 5 FOLLOWER]: Advancing to term 6
21:09:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:58.556735 17212 leader_election.cc:336] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 election: RPC error from VoteRequest() call to peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111)
21:09:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.558168 17392 leader_election.cc:290] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 election: Requested vote from peers b42c9a70f6c1438385ef2f0d250f8fc8 (127.15.79.61:40625), 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573)
21:09:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.559666 17332 raft_consensus.cc:2463] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 6 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ad20382b782c44a3a984136f62be401f in term 6.
21:09:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.560906 17214 leader_election.cc:304] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [CANDIDATE]: Term 6 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ad20382b782c44a3a984136f62be401f, b42c9a70f6c1438385ef2f0d250f8fc8; no voters: 6e646947c6524250b8154c630cc52ebb
21:09:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.563108 17392 raft_consensus.cc:2798] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 6 FOLLOWER]: Leader election won for term 6
21:09:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.571568 17392 raft_consensus.cc:695] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [term 6 LEADER]: Becoming Leader. State: Replica: ad20382b782c44a3a984136f62be401f, State: Running, Role: LEADER
21:09:58.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.572609 17392 consensus_queue.cc:237] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 5.8, Last appended by leader: 8, Current term: 6, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } }
21:09:58.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.596624 17397 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ad20382b782c44a3a984136f62be401f. Latest consensus state: current_term: 6 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:58.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.597620 17397 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: LEADER
21:09:58.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.617336 17399 catalog_manager.cc:1476] Loading table and tablet metadata into memory...
21:09:58.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.625586 17399 catalog_manager.cc:1485] Initializing Kudu cluster ID...
21:09:58.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.636462 17399 catalog_manager.cc:1260] Loaded cluster ID: d885459620e04d6ea5a01814bc6bdaea
21:09:58.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.636914 17399 catalog_manager.cc:1496] Initializing Kudu internal certificate authority...
21:09:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.649595 17399 catalog_manager.cc:1505] Loading token signing keys...
21:09:58.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.653861 17399 catalog_manager.cc:5910] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f: Loaded TSK: 0
21:09:58.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.655218 17399 catalog_manager.cc:1515] Initializing in-progress tserver states...
21:09:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.565811 17387 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:58.877558 17386 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1315 milliseconds
21:09:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.877777 17359 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:09:58.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:57.563123 17384 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:09:58.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.884590 17359 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:09:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.887737 17359 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:09:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.889251 17359 hybrid_clock.cc:648] HybridClock initialized: now 1736543398889194 us; error 66 us; skew 500 ppm
21:09:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.890437 17359 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:09:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jan 10 21:09:58 dist-test-slave-4hht krb5kdc[15687](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1736543398, etypes {rep=17 tkt=17 ses=17}, kudu/127.15.79.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:09:58.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.897902 17359 init.cc:377] Logged in from keytab as kudu/127.15.79.60@KRBTEST.COM (short username kudu)
21:09:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.903501 17359 webserver.cc:458] Webserver started at http://127.15.79.60:42079/ using document root <none> and password file <none>
21:09:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.905476 17359 fs_manager.cc:362] Metadata directory not provided
21:09:58.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.905740 17359 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:09:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.915725 17359 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
21:09:58.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.919973 17405 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:09:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.921157 17359 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:09:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.921525 17359 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data,/tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e646947c6524250b8154c630cc52ebb"
21:09:58.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-01-10 21:09:24 on dist-test-slave-4hht"
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.923759 17359 fs_report.cc:389] FS layout report
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/wal
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/data
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:09:58.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:09:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:09:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:09:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:09:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:09:58.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.979274 17359 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:09:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.985214 17359 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:09:58.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:58.985641 17359 kserver.cc:163] Server-wide thread pool size limit: 3276
21:09:59.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.030088 17332 raft_consensus.cc:1270] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [term 6 FOLLOWER]: Refusing update from remote peer ad20382b782c44a3a984136f62be401f: Log matching property violated. Preceding OpId in replica: term: 5 index: 8. Preceding OpId from leader: term: 6 index: 9. (index mismatch)
21:09:59.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.032315 17397 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [LEADER]: Connected to new peer: Peer: permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.001s
21:09:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:59.081465 17212 consensus_peers.cc:487] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f -> Peer 6e646947c6524250b8154c630cc52ebb (127.15.79.60:43573): Couldn't send request to peer 6e646947c6524250b8154c630cc52ebb. Status: Network error: Client connection negotiation failed: client connection to 127.15.79.60:43573: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:09:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.089293 17432 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ad20382b782c44a3a984136f62be401f. Latest consensus state: current_term: 6 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.090225 17432 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.093380 17432 sys_catalog.cc:455] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 6 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.094978 17432 sys_catalog.cc:458] T 00000000000000000000000000000000 P b42c9a70f6c1438385ef2f0d250f8fc8 [sys.catalog]: This master's current role is: FOLLOWER
21:09:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.097277 17392 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 6 leader_uuid: "ad20382b782c44a3a984136f62be401f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad20382b782c44a3a984136f62be401f" member_type: VOTER last_known_addr { host: "127.15.79.62" port: 39595 } } peers { permanent_uuid: "b42c9a70f6c1438385ef2f0d250f8fc8" member_type: VOTER last_known_addr { host: "127.15.79.61" port: 40625 } } peers { permanent_uuid: "6e646947c6524250b8154c630cc52ebb" member_type: VOTER last_known_addr { host: "127.15.79.60" port: 43573 } } }
21:09:59.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.098091 17392 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad20382b782c44a3a984136f62be401f [sys.catalog]: This master's current role is: LEADER
21:09:59.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.128057 17359 rpc_server.cc:307] RPC server started. Bound to: 127.15.79.60:43573
21:09:59.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.128234 17462 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.15.79.60:43573 every 8 connection(s)
21:09:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.131757 17359 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/master-2/data/info.pb
21:09:59.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.135365 15676 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu as pid 17359
21:09:59.136 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:09:59.136 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.15.79.61:40625(127.15.79.61:40625)] connecting to peer
21:09:59.139 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.15.79.61:40625(127.15.79.61:40625)] Successfully connected to peer
21:09:59.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.143950 17463 sys_catalog.cc:263] Verifying existing consensus state
21:09:59.146 [DEBUG - kudu-nio-0] (SecurityContext.java:225) Refreshing Kerberos credentials...
21:09:59.147 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskOzYfYq/test-tmp/mini-kudu-cluster7131829176226720979/krb5kdc/krb5cc
21:09:59.148 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:09:59.149 [WARN - kudu-nio-0] (SecurityContext.java:234) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
21:09:59.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.15.79.61:40625(127.15.79.61:40625)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:09:59.150 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.15.79.61:40625(127.15.79.61:40625)] handling channelInactive
21:09:59.150 [DEBUG - main] (Connection.java:575) [peer master-127.15.79.61:40625(127.15.79.61:40625)] disconnecting while in state TERMINATED
21:09:59.151 [DEBUG - main] (Connection.java:575) [peer master-127.15.79.62:39595(127.15.79.62:39595)] disconnecting while in state TERMINATED
21:09:59.152 [DEBUG - main] (Connection.java:575) [peer master-127.15.79.60:43573(127.15.79.60:43573)] disconnecting while in state TERMINATED
21:09:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.150723 17463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap starting.
21:09:59.152 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250110 21:09:59.153053 17465 negotiation.cc:337] Failed RPC negotiation. Trace:
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.138333 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57690 (local address 127.15.79.61:40625)
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145391 (+ 7058us) server_negotiation.cc:206] Beginning negotiation
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145426 (+ 35us) server_negotiation.cc:399] Waiting for connection header
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145476 (+ 50us) server_negotiation.cc:407] Connection header received
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145618 (+ 142us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145644 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
21:09:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.145852 (+ 208us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
21:09:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.146040 (+ 188us) server_negotiation.cc:226] Negotiated authn=SASL
21:09:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0110 21:09:59.150579 (+ 4539us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57690: BlockingRecv error: recv got EOF from 127.0.0.1:57690 (error 108)
21:09:59.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":6443,"thread_start_us":6122,"threads_started":1}
21:09:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.171322 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 17193
21:09:59.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.195381 17463 log.cc:826] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Log is configured to *not* fsync() on all Append() calls
21:09:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.208846 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 17265
21:09:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.215211 17463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:09:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.215961 17463 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e646947c6524250b8154c630cc52ebb: Bootstrap complete.
21:09:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:09:59.232736 15676 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 17359
21:10:00.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:10:00Z chronyd exiting
21:10:01.362 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:10:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.363893 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15238
21:10:01.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.384727 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15394
21:10:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.408596 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15532
21:10:01.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.429435 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 14967
21:10:01.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.454348 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15037
21:10:01.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250110 21:10:01.481015 14955 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskOzYfYq/build/tsan/bin/kudu with pid 15112
21:10:02.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-01-10T21:10:02Z chronyd exiting
21:10:03.557 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more