15:08:07.233 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:08:07.248 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwnAIsP/test-tmp
15:08:07.255 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:08:07.258 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwnAIsP/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:08:07.441 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:08:07.632 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:08:07.634 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:08:07.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:07.639511 1508 mini_oidc.cc:150] Starting JWKS server
15:08:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:07.643760 1508 webserver.cc:577] Webserver started at http://127.0.0.1:39249/ using document root <none> and password file <none>
15:08:07.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:07.644375 1508 mini_oidc.cc:186] Starting OIDC Discovery server
15:08:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:07.646507 1508 webserver.cc:577] Webserver started at http://127.0.0.1:37363/ using document root <none> and password file <none>
15:08:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T15:08:07Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:08:07.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T15:08:07Z Disabled control of system clock
15:08:07.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:07.709708 1508 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal
15:08:07.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data
15:08:07.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:08:07.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:08:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:08:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:07.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/logs
15:08:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data/info.pb
15:08:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:08:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:08:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:08:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:08:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:07.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:08:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:08:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.62:35617
15:08:07.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.62
15:08:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:07.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:07.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:07.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json with env {}
15:08:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.026505 1520 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
15:08:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.027063 1520 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:08:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.027964 1520 flags.cc:432] Enabled unsafe flag: --never_fsync=true
15:08:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.062924 1520 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
15:08:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.063242 1520 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
15:08:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.063450 1520 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
15:08:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.063652 1520 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
15:08:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.063890 1520 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
15:08:08.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.064128 1520 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39249/jwks.json
15:08:08.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:08.108243 1520 master_runner.cc:387] Master server non-default flags:
15:08:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/logs
15:08:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:08:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:08.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:08.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:08.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.62:35617
15:08:08.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data/info.pb
15:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.62
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:08:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:08:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:08:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
15:08:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:08:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 14:43:24 UTC on e7f111948823
15:08:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14846
15:08:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:08:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:08.109557 1520 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:08:08.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:08.111698 1520 file_cache.cc:504] Constructed file cache file cache with capacity 419430
15:08:08.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.126312 1526 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:09.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:08.126300 1527 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:09.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.408592 1529 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.410980 1528 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1279 milliseconds
15:08:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.411831 1520 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.286s user 0.428s sys 0.849s
15:08:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.412153 1520 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.287s user 0.428s sys 0.849s
15:08:09.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.412451 1520 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:08:09.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.413899 1520 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:08:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.420707 1520 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:08:09.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.422113 1520 hybrid_clock.cc:648] HybridClock initialized: now 1795014489422066 us; error 65 us; skew 500 ppm
15:08:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.423202 1520 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:08:09.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.430864 1520 webserver.cc:577] Webserver started at http://127.1.121.62:44511/ using document root <none> and password file <none>
15:08:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.432158 1520 fs_manager.cc:362] Metadata directory not provided
15:08:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.432444 1520 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:08:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.433044 1520 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
15:08:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.438103 1520 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data/instance:
15:08:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a80a404527d4435a596780c8c01808b"
15:08:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:09 on dist-test-slave-snf6"
15:08:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.439790 1520 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal/instance:
15:08:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a80a404527d4435a596780c8c01808b"
15:08:09.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:09 on dist-test-slave-snf6"
15:08:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.448432 1520 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
15:08:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.455710 1536 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:08:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.457180 1520 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.003s
15:08:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.457559 1520 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data,/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal
15:08:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7a80a404527d4435a596780c8c01808b"
15:08:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:09 on dist-test-slave-snf6"
15:08:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.458007 1520 fs_report.cc:389] FS layout report
15:08:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:08:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal
15:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal
15:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data/data
15:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:08:09.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:08:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:08:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:08:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:08:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:08:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:08:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:08:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:08:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:08:09.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.528620 1520 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:08:09.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.530367 1520 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:08:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.530932 1520 kserver.cc:163] Server-wide thread pool size limit: 3276
15:08:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.618966 1520 rpc_server.cc:307] RPC server started. Bound to: 127.1.121.62:35617
15:08:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.619051 1587 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.121.62:35617 every 8 connection(s)
15:08:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.621831 1520 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/data/info.pb
15:08:09.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.627908 1588 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:08:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.631147 1508 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu as pid 1520
15:08:09.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.631510 1508 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-0/wal/instance
15:08:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.634902 1508 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:09.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal
15:08:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data
15:08:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:08:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:08:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:08:09.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/logs
15:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data/info.pb
15:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.61:44715
15:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.61
15:08:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json with env {}
15:08:09.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.642261 1588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } has no permanent_uuid. Determining permanent_uuid...
15:08:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.675300 1588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } has no permanent_uuid. Determining permanent_uuid...
15:08:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.678165 1539 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.682780 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.732347 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 1
15:08:09.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.736830 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:09.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.805326 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 2
15:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.810123 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:09.901536 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 3
15:08:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.906129 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:09.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.955192 1590 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
15:08:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.955816 1590 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:08:09.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.956455 1590 flags.cc:432] Enabled unsafe flag: --never_fsync=true
15:08:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.989919 1590 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
15:08:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.990238 1590 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
15:08:09.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.990476 1590 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
15:08:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.990682 1590 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
15:08:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.990875 1590 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
15:08:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:09.991021 1590 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39249/jwks.json
15:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.031762 1590 master_runner.cc:387] Master server non-default flags:
15:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/logs
15:08:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.61:44715
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json
15:08:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data/info.pb
15:08:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.61
15:08:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:08:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:08:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:08:10.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:08:10.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 14:43:24 UTC on e7f111948823
15:08:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14846
15:08:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:08:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.033179 1590 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:08:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.034914 1590 file_cache.cc:504] Constructed file cache file cache with capacity 419430
15:08:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.049594 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 4
15:08:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:10.050041 1599 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:10.058693 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.358206 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 5
15:08:10.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:10.363197 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:10.910579 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 6
15:08:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:10.917984 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.61:44715: Network error: Client connection negotiation failed: client connection to 127.1.121.61:44715: connect: Connection refused (error 111)
15:08:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:10.053242 1600 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.258060 1601 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:08:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.259845 1602 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.259857 1590 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:08:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.262960 1590 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:08:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.265935 1590 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:08:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.267346 1590 hybrid_clock.cc:648] HybridClock initialized: now 1795014491267269 us; error 80 us; skew 500 ppm
15:08:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.268409 1590 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:08:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.274534 1590 webserver.cc:577] Webserver started at http://127.1.121.61:45323/ using document root <none> and password file <none>
15:08:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.275408 1590 fs_manager.cc:362] Metadata directory not provided
15:08:11.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.275589 1590 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:08:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.276101 1590 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
15:08:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.280550 1590 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data/instance:
15:08:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:11 on dist-test-slave-snf6"
15:08:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.281833 1590 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal/instance:
15:08:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:11 on dist-test-slave-snf6"
15:08:11.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.289951 1590 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
15:08:11.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.296360 1611 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:08:11.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.297600 1590 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:08:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.297905 1590 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data,/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal
15:08:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:11 on dist-test-slave-snf6"
15:08:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.298233 1590 fs_report.cc:389] FS layout report
15:08:11.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data/data
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:08:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:08:11.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:08:11.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.359239 1590 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:08:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.360811 1590 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:08:11.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.361241 1590 kserver.cc:163] Server-wide thread pool size limit: 3276
15:08:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.437279 1590 rpc_server.cc:307] RPC server started. Bound to: 127.1.121.61:44715
15:08:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.437457 1662 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.121.61:44715 every 8 connection(s)
15:08:11.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.440781 1590 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/data/info.pb
15:08:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.448196 1663 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:08:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.450937 1508 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu as pid 1590
15:08:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.451501 1508 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-1/wal/instance
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.454911 1508 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:08:11.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/logs
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data/info.pb
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:08:11.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.60:42157
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.60
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:11.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json with env {}
15:08:11.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.464989 1663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } has no permanent_uuid. Determining permanent_uuid...
15:08:11.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.486627 1663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } has no permanent_uuid. Determining permanent_uuid...
15:08:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.501446 1663 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } has no permanent_uuid. Determining permanent_uuid...
15:08:11.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.503899 1612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.508172 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.557636 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 1
15:08:11.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.562667 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.631055 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 2
15:08:11.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.636754 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.728230 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 3
15:08:11.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.732434 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.777915 1665 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
15:08:11.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.778478 1665 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:08:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.778999 1665 flags.cc:432] Enabled unsafe flag: --never_fsync=true
15:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.812826 1665 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
15:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.813200 1665 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
15:08:11.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.813527 1665 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
15:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.813813 1665 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
15:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.814143 1665 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
15:08:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.814323 1665 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39249/jwks.json
15:08:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.851763 1665 master_runner.cc:387] Master server non-default flags:
15:08:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/logs
15:08:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:08:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.121.20:46289
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal
15:08:11.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.121.62:35617,127.1.121.61:44715,127.1.121.60:42157
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.121.60:42157
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39249/jwks.json
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:08:11.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data/info.pb
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.121.60
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:08:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 14:43:24 UTC on e7f111948823
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14846
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.853453 1665 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:08:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.855643 1665 file_cache.cc:504] Constructed file cache file cache with capacity 419430
15:08:11.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.867123 1675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.875882 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 4
15:08:11.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.883493 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:11.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:11.978628 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } attempt: 7
15:08:12.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.016975 1588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } has no permanent_uuid. Determining permanent_uuid...
15:08:12.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.039577 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.100257 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 1
15:08:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.119419 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.183032 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 5
15:08:12.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.191977 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.201004 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 2
15:08:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.209132 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.294652 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 3
15:08:12.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.302757 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.443290 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 4
15:08:12.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.455920 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.739501 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 6
15:08:12.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:12.739508 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 5
15:08:12.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.749279 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:12.773559 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.301823 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 6
15:08:13.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.333633 1588 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.272331 1674 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 1665
15:08:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.690757 1665 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.822s user 0.602s sys 1.219s
15:08:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.691126 1665 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.823s user 0.602s sys 1.219s
15:08:13.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:11.869019 1676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.693231 1678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:08:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.695993 1677 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1823 milliseconds
15:08:13.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.696017 1665 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:08:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.697400 1665 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:08:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.699914 1665 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:08:13.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.701303 1665 hybrid_clock.cc:648] HybridClock initialized: now 1795014493701251 us; error 51 us; skew 500 ppm
15:08:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.702235 1665 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:08:13.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.708380 1665 webserver.cc:577] Webserver started at http://127.1.121.60:44113/ using document root <none> and password file <none>
15:08:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.709436 1665 fs_manager.cc:362] Metadata directory not provided
15:08:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.709693 1665 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:08:13.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.710246 1665 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
15:08:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.714596 1665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data/instance:
15:08:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6074ab3567be4f298ece03d26ab4f6db"
15:08:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:13 on dist-test-slave-snf6"
15:08:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.715986 1665 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal/instance:
15:08:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6074ab3567be4f298ece03d26ab4f6db"
15:08:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:13 on dist-test-slave-snf6"
15:08:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.723724 1665 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.004s
15:08:13.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.729537 1691 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:08:13.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.730633 1665 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
15:08:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.730988 1665 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data,/tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal
15:08:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6074ab3567be4f298ece03d26ab4f6db"
15:08:13.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 15:08:13 on dist-test-slave-snf6"
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.731396 1665 fs_report.cc:389] FS layout report
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data/data
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:08:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:08:13.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:08:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.789222 1665 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:08:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.790889 1665 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:08:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.791425 1665 kserver.cc:163] Server-wide thread pool size limit: 3276
15:08:13.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.834085 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 7
15:08:13.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:13.841221 1663 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.121.60:42157: Network error: Client connection negotiation failed: client connection to 127.1.121.60:42157: connect: Connection refused (error 111)
15:08:13.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.866954 1665 rpc_server.cc:307] RPC server started. Bound to: 127.1.121.60:42157
15:08:13.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.867014 1744 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.121.60:42157 every 8 connection(s)
15:08:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.870322 1665 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/data/info.pb
15:08:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.876349 1745 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:08:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.880743 1508 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnAIsP/build/tsan/bin/kudu as pid 1665
15:08:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.881234 1508 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnAIsP/test-tmp/mini-kudu-cluster8373829225223238134/master-2/wal/instance
15:08:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.890157 1745 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } has no permanent_uuid. Determining permanent_uuid...
15:08:13.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.915019 1745 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } has no permanent_uuid. Determining permanent_uuid...
15:08:13.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.927802 1745 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } has no permanent_uuid. Determining permanent_uuid...
15:08:13.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.943522 1745 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: Bootstrap starting.
15:08:13.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.949592 1745 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: Neither blocks nor log segments found. Creating new log.
15:08:13.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.952544 1745 log.cc:826] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: Log is configured to *not* fsync() on all Append() calls
15:08:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.957988 1745 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: No bootstrap required, opened a new log
15:08:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.979549 1745 raft_consensus.cc:360] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.980373 1745 raft_consensus.cc:386] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:08:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.980662 1745 raft_consensus.cc:748] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6074ab3567be4f298ece03d26ab4f6db, State: Initialized, Role: FOLLOWER
15:08:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.981549 1745 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.984614 1755 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } } }
15:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.985437 1755 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [sys.catalog]: This master's current role is: FOLLOWER
15:08:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:13.986439 1745 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [sys.catalog]: configured and running, proceeding with master startup.
15:08:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.013837 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:14.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.014223 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:14.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.007030 1665 master_runner.cc:187] Error getting master registration for 127.1.121.62:35617: OK, instance_id { permanent_uuid: "7a80a404527d4435a596780c8c01808b" instance_seqno: 1795014489598652 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:08:14.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.031375 1665 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
15:08:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.046792 1697 tablet.cc:2441] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:08:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.367185 1588 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 7
15:08:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.388447 1588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: Bootstrap starting.
15:08:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.396303 1588 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: Neither blocks nor log segments found. Creating new log.
15:08:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.399083 1588 log.cc:826] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: Log is configured to *not* fsync() on all Append() calls
15:08:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.404232 1588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: No bootstrap required, opened a new log
15:08:14.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.432698 1588 raft_consensus.cc:360] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.433538 1588 raft_consensus.cc:386] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:08:14.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.433816 1588 raft_consensus.cc:748] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7a80a404527d4435a596780c8c01808b, State: Initialized, Role: FOLLOWER
15:08:14.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.434782 1588 consensus_queue.cc:261] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.437674 1770 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } } }
15:08:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.438691 1770 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [sys.catalog]: This master's current role is: FOLLOWER
15:08:14.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.439674 1588 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [sys.catalog]: configured and running, proceeding with master startup.
15:08:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.468473 1755 raft_consensus.cc:499] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:08:14.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.469053 1755 raft_consensus.cc:521] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.476729 1755 leader_election.cc:290] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7a80a404527d4435a596780c8c01808b (127.1.121.62:35617), a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715)
15:08:14.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.477157 1770 raft_consensus.cc:499] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:08:14.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.477648 1770 raft_consensus.cc:521] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.480712 1563 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6074ab3567be4f298ece03d26ab4f6db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7a80a404527d4435a596780c8c01808b" is_pre_election: true
15:08:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.481441 1563 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6074ab3567be4f298ece03d26ab4f6db in term 0.
15:08:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.483691 1718 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6074ab3567be4f298ece03d26ab4f6db" is_pre_election: true
15:08:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.484421 1718 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7a80a404527d4435a596780c8c01808b in term 0.
15:08:14.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.486734 1781 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:14.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.487057 1781 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:14.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.487957 1537 leader_election.cc:304] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6074ab3567be4f298ece03d26ab4f6db, 7a80a404527d4435a596780c8c01808b; no voters:
15:08:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.501048 1770 leader_election.cc:290] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715), 6074ab3567be4f298ece03d26ab4f6db (127.1.121.60:42157)
15:08:14.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.501050 1695 leader_election.cc:304] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6074ab3567be4f298ece03d26ab4f6db, 7a80a404527d4435a596780c8c01808b; no voters:
15:08:14.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.501998 1755 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Leader pre-election won for term 1
15:08:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.502327 1755 raft_consensus.cc:499] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:08:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.502650 1755 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 0 FOLLOWER]: Advancing to term 1
15:08:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.503350 1770 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Leader pre-election won for term 1
15:08:14.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.509552 1755 raft_consensus.cc:521] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.512766 1563 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6074ab3567be4f298ece03d26ab4f6db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7a80a404527d4435a596780c8c01808b"
15:08:14.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.513305 1563 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 0 FOLLOWER]: Advancing to term 1
15:08:14.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.515523 1755 leader_election.cc:290] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [CANDIDATE]: Term 1 election: Requested vote from peers 7a80a404527d4435a596780c8c01808b (127.1.121.62:35617), a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715)
15:08:14.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.521351 1563 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6074ab3567be4f298ece03d26ab4f6db in term 1.
15:08:14.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.522063 1770 raft_consensus.cc:499] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:08:14.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.522419 1770 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 1 FOLLOWER]: Advancing to term 2
15:08:14.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.522918 1695 leader_election.cc:304] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6074ab3567be4f298ece03d26ab4f6db, 7a80a404527d4435a596780c8c01808b; no voters:
15:08:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.523766 1755 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 FOLLOWER]: Leader election won for term 1
15:08:14.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.533401 1770 raft_consensus.cc:521] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:14.544456 1542 tablet.cc:2441] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:08:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.558934 1718 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6074ab3567be4f298ece03d26ab4f6db"
15:08:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.557917 1770 leader_election.cc:290] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 2 election: Requested vote from peers a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715), 6074ab3567be4f298ece03d26ab4f6db (127.1.121.60:42157)
15:08:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.570410 1755 raft_consensus.cc:705] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 LEADER]: Becoming Leader. State: Replica: 6074ab3567be4f298ece03d26ab4f6db, State: Running, Role: LEADER
15:08:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.571961 1755 consensus_queue.cc:238] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.496258 1520 master_runner.cc:187] Error getting master registration for 127.1.121.61:44715: OK, instance_id { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" instance_seqno: 1795014491416295 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:08:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:14.677335 1520 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
15:08:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.029894 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.030328 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.491631 1781 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.492650 1781 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:15.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.905596 1674 debug-util.cc:398] Leaking SignalData structure 0x7b0800029400 after lost signal to thread 1755
15:08:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:15.906097 1674 kernel_stack_watchdog.cc:198] Thread 1755 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 308ms:
15:08:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:08:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:08:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:08:15.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:08:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:15.931622 1663 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } attempt: 8
15:08:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:16.027161 1539 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.1.121.61:44715 timed out after 1.540s (SENT) [suppressed 13 similar messages]
15:08:16.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:16.034385 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:16.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:16.034716 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:16.498622 1781 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:16.499064 1781 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:17.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.038784 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:17.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.039136 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:17.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.108340 1674 debug-util.cc:398] Leaking SignalData structure 0x7b0800037760 after lost signal to thread 1694
15:08:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.108820 1674 kernel_stack_watchdog.cc:198] Thread 1694 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 983ms:
15:08:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:08:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:08:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:08:17.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:08:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.504786 1781 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.505218 1781 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:17.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.580091 1539 leader_election.cc:336] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715): Timed out: RequestConsensusVote RPC to 127.1.121.61:44715 timed out after 1.540s (SENT)
15:08:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:17.848974 1539 leader_election.cc:336] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a921f0c73f4c4e058178e89dd7b0b3fc (127.1.121.61:44715): Timed out: RequestConsensusVote RPC to 127.1.121.61:44715 timed out after 1.643s (SENT)
15:08:17.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:17.849495 1539 leader_election.cc:304] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7a80a404527d4435a596780c8c01808b; no voters: 6074ab3567be4f298ece03d26ab4f6db, a921f0c73f4c4e058178e89dd7b0b3fc
15:08:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:17.864269 1792 raft_consensus.cc:2771] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
15:08:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.043550 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.043926 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.109480 1674 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 1755
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.109947 1674 kernel_stack_watchdog.cc:198] Thread 1755 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 1511ms:
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:08:18.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:08:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.426767 1784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 6074ab3567be4f298ece03d26ab4f6db. Latest consensus state: current_term: 1 leader_uuid: "6074ab3567be4f298ece03d26ab4f6db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } } }
15:08:18.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.434321 1784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [sys.catalog]: This master's current role is: LEADER
15:08:18.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.437706 1563 raft_consensus.cc:1250] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b [term 2 FOLLOWER]: Rejecting Update request from peer 6074ab3567be4f298ece03d26ab4f6db for earlier term 1. Current term is 2. Ops: []
15:08:18.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.431105 1755 thread.cc:642] raft [worker] (thread pool) Time spent creating pthread: real 3.834s user 0.670s sys 1.465s
15:08:18.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.441789 1693 thread.cc:642] raft [worker] (thread pool) Time spent creating pthread: real 3.381s user 0.004s sys 0.000s
15:08:18.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.452008 1755 thread.cc:609] raft [worker] (thread pool) Time spent starting thread: real 3.855s user 0.671s sys 1.465s
15:08:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.442312 1694 thread.cc:642] raft [worker] (thread pool) Time spent creating pthread: real 3.318s user 0.000s sys 0.004s
15:08:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.452894 1694 thread.cc:609] raft [worker] (thread pool) Time spent starting thread: real 3.328s user 0.001s sys 0.004s
15:08:18.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.452739 1787 consensus_queue.cc:1060] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 }, Status: INVALID_TERM, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.453656 1784 raft_consensus.cc:3091] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 LEADER]: Stepping down as leader of term 1
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.453996 1784 raft_consensus.cc:748] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6074ab3567be4f298ece03d26ab4f6db, State: Running, Role: LEADER
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.454406 1694 connection.cc:256] RPC call timeout handler was delayed by -2.4369224160000158s: this may be due to a process-wide pause such as swapping, logging-related delays, or allocator lock contention. Will allow extra 0.17170000000000002s for a response
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.454828 1694 connection.cc:256] RPC call timeout handler was delayed by -2.3691641890000028s: this may be due to a process-wide pause such as swapping, logging-related delays, or allocator lock contention. Will allow extra 0.1744s for a response
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.454886 1784 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.452099 1693 thread.cc:609] raft [worker] (thread pool) Time spent starting thread: real 3.391s user 0.006s sys 0.000s
15:08:18.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.456108 1784 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db [term 1 FOLLOWER]: Advancing to term 2
15:08:18.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.449726 1794 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
15:08:18.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.475086 1794 catalog_manager.cc:1535] Initializing Kudu cluster ID...
15:08:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.486616 1794 catalog_manager.cc:1511] Initializing Kudu cluster ID failed; change in term detected: 1 vs 2: Illegal state: replica 6074ab3567be4f298ece03d26ab4f6db is not leader of this config: current role FOLLOWER
15:08:18.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.499171 1663 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc: Bootstrap starting.
15:08:18.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.512063 1781 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 7a80a404527d4435a596780c8c01808b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:18.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.512544 1781 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.515146 1663 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc: Neither blocks nor log segments found. Creating new log.
15:08:18.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.517673 1663 log.cc:826] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc: Log is configured to *not* fsync() on all Append() calls
15:08:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.530563 1663 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc: No bootstrap required, opened a new log
15:08:18.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.563449 1663 raft_consensus.cc:360] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.564997 1663 raft_consensus.cc:386] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:08:18.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.565605 1663 raft_consensus.cc:748] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a921f0c73f4c4e058178e89dd7b0b3fc, State: Initialized, Role: FOLLOWER
15:08:18.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.567168 1663 consensus_queue.cc:261] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } }
15:08:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.581228 1797 sys_catalog.cc:455] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7a80a404527d4435a596780c8c01808b" member_type: VOTER last_known_addr { host: "127.1.121.62" port: 35617 } } peers { permanent_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" member_type: VOTER last_known_addr { host: "127.1.121.61" port: 44715 } } peers { permanent_uuid: "6074ab3567be4f298ece03d26ab4f6db" member_type: VOTER last_known_addr { host: "127.1.121.60" port: 42157 } } }
15:08:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.582487 1797 sys_catalog.cc:458] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [sys.catalog]: This master's current role is: FOLLOWER
15:08:18.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.583376 1663 sys_catalog.cc:565] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [sys.catalog]: configured and running, proceeding with master startup.
15:08:18.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.583526 1638 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6074ab3567be4f298ece03d26ab4f6db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" is_pre_election: true
15:08:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.584364 1638 raft_consensus.cc:2490] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6074ab3567be4f298ece03d26ab4f6db in term 0.
15:08:18.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.584904 1638 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47202 (request call id 1) took 4110 ms (client timeout 1717 ms). Trace:
15:08:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.585249 1638 rpcz_store.cc:269] 1118 15:08:14.474744 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:14.475399 (+ 655us) service_pool.cc:224] Handling call
15:08:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.584835 (+4109436us) inbound_call.cc:177] Queueing success response
15:08:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.587373 1637 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" is_pre_election: true
15:08:18.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.588317 1637 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47182 (request call id 2) took 4102 ms (client timeout 1540 ms). Trace:
15:08:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.588670 1637 rpcz_store.cc:269] 1118 15:08:14.486234 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:14.486371 (+ 137us) service_pool.cc:224] Handling call
15:08:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.588258 (+4101887us) inbound_call.cc:177] Queueing success response
15:08:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.589581 1633 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:18.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.590426 1633 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47182 (request call id 5) took 2384 ms (client timeout 1643 ms). Trace:
15:08:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.590803 1633 rpcz_store.cc:269] 1118 15:08:16.205745 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:16.205878 (+ 133us) service_pool.cc:224] Handling call
15:08:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.590369 (+2384491us) inbound_call.cc:177] Queueing success response
15:08:18.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.591851 1632 raft_consensus.cc:3096] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc [term 0 FOLLOWER]: Advancing to term 1
15:08:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.592689 1635 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.596585 1635 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47182 (request call id 3) took 4042 ms (client timeout 1643 ms). Trace:
15:08:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.597817 1635 rpcz_store.cc:269] 1118 15:08:14.554181 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:14.554292 (+ 111us) service_pool.cc:224] Handling call
15:08:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.596525 (+4042233us) inbound_call.cc:177] Queueing success response
15:08:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.597333 1634 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7a80a404527d4435a596780c8c01808b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc" is_pre_election: true
15:08:18.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.600178 1634 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47182 (request call id 4) took 2557 ms (client timeout 1540 ms). Trace:
15:08:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.600750 1634 rpcz_store.cc:269] 1118 15:08:16.042967 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:16.043099 (+ 132us) service_pool.cc:224] Handling call
15:08:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.600105 (+2557006us) inbound_call.cc:177] Queueing success response
15:08:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.593321 1636 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "6074ab3567be4f298ece03d26ab4f6db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a921f0c73f4c4e058178e89dd7b0b3fc"
15:08:18.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.612747 1636 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:47202 (request call id 2) took 4097 ms (client timeout 1744 ms). Trace:
15:08:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.614264 1636 rpcz_store.cc:269] 1118 15:08:14.515797 (+ 0us) service_pool.cc:167] Inserting onto call queue
15:08:18.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:14.515938 (+ 141us) service_pool.cc:224] Handling call
15:08:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 15:08:18.612685 (+4096747us) inbound_call.cc:177] Queueing success response
15:08:18.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:08:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:18.629606 1617 tablet.cc:2441] T 00000000000000000000000000000000 P a921f0c73f4c4e058178e89dd7b0b3fc: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:08:18.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.698691 1808 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:08:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.715508 1590 master_runner.cc:264] No leader master found from master a921f0c73f4c4e058178e89dd7b0b3fc
15:08:18.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:18.715783 1590 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
15:08:18.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) F20261118 15:08:18.897210 1508 external_mini_cluster.cc:656] Check failed: leader_master() != nullptr Must have started at least 1 master before adding tablet servers
15:08:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) *** Check failure stack trace: ***
15:08:18.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97a121a920 google::LogMessage::Flush()
15:08:18.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97a121fa5b google::LogMessageFatal::~LogMessageFatal()
15:08:18.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97aaa9ddbd kudu::cluster::ExternalMiniCluster::AddTabletServer()
15:08:18.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97aaa9be3b kudu::cluster::ExternalMiniCluster::Start()
15:08:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab1f773a kudu::tools::(anonymous namespace)::ProcessRequest()
15:08:18.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab1f5fa7 kudu::tools::(anonymous namespace)::RunControlShell()
15:08:18.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab0bfc02 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_
15:08:18.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab0bfb6d std::__1::__invoke_void_return_wrapper<>::__call<>()
15:08:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab0bfadd std::__1::__function::__alloc_func<>::operator()()
15:08:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97ab0be746 std::__1::__function::__func<>::operator()()
15:08:19.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97aac48770 std::__1::__function::__value_func<>::operator()()
15:08:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97aac3ace2 std::__1::function<>::operator()()
15:08:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:19.044291 1665 master_runner.cc:233] Existing masters have differing terms: 2 vs 1
15:08:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 15:08:19.044770 1665 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
15:08:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:19.047016 1766 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6074ab3567be4f298ece03d26ab4f6db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:08:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 15:08:19.047254 1766 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:08:19.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f97aac38993 kudu::tools::Action::Run()
15:08:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x324911 kudu::tools::DispatchCommand()
15:08:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x325248 kudu::tools::RunTool()
15:08:19.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x325672 main
15:08:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x7f979e2d0c87 __libc_start_main
15:08:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) @ 0x26efba _start
15:08:19.321 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestPartitionPruner.testPrimaryKeyPrefixRangePruning: failed attempt 1
java.io.EOFException
at java.base/java.io.DataInputStream.readInt(DataInputStream.java:386)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.io.EOFException
at java.base/java.io.DataInputStream.readInt(DataInputStream.java:386)
at org.apache.kudu.test.cluster.MiniKuduCluster.sendRequestToCluster(MiniKuduCluster.java:191)
at org.apache.kudu.test.cluster.MiniKuduCluster.start(MiniKuduCluster.java:266)
at org.apache.kudu.test.cluster.MiniKuduCluster$MiniKuduClusterBuilder.build(MiniKuduCluster.java:897)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:156)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)