08:06:03.196 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:06:03.211 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskA19hJ6/test-tmp
08:06:03.213 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:06:03.216 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskA19hJ6/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:06:03.456 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:06:03.660 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:06:03.661 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:06:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:03.664664 14049 mini_oidc.cc:150] Starting JWKS server
08:06:03.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:03.667996 14049 webserver.cc:480] Webserver started at http://127.0.0.1:41883/ using document root <none> and password file <none>
08:06:03.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:03.668493 14049 mini_oidc.cc:186] Starting OIDC Discovery server
08:06:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:03.670162 14049 webserver.cc:480] Webserver started at http://127.0.0.1:40569/ using document root <none> and password file <none>
08:06:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:03Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:06:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:03Z Disabled control of system clock
08:06:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:03.725620 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal
08:06:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data
08:06:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:03.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/logs
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data/info.pb
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.126:40703
08:06:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.126
08:06:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:03.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json with env {}
08:06:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:03.987713 14061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:03.988184 14061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:03.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:03.988705 14061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.016067 14061 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.016304 14061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.016505 14061 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.016723 14061 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.016893 14061 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.017066 14061 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41883/jwks.json
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:04.047839 14061 master_runner.cc:386] Master server non-default flags:
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:04.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data
08:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal
08:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:04.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.126:40703
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data/info.pb
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.126
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/logs
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:04.049031 14061 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:04.050629 14061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.062736 14067 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:04.063544 14068 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.636987 14061 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.574s user 0.588s sys 0.986s
08:06:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.637656 14061 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.575s user 0.588s sys 0.986s
08:06:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.465796 14066 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 14061
08:06:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.639565 14070 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:05.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.641819 14069 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1577 milliseconds
08:06:05.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.641834 14061 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:05.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.643026 14061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:05.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.645226 14061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:05.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.646521 14061 hybrid_clock.cc:648] HybridClock initialized: now 1756800365646479 us; error 52 us; skew 500 ppm
08:06:05.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.647184 14061 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:05.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.652500 14061 webserver.cc:480] Webserver started at http://127.13.184.126:45679/ using document root <none> and password file <none>
08:06:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.653268 14061 fs_manager.cc:362] Metadata directory not provided
08:06:05.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.653438 14061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:05.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.653951 14061 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.657709 14061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data/instance:
08:06:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7841f31884c4209acb421eb987aa665"
08:06:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:05 on dist-test-slave-r4nz"
08:06:05.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.658680 14061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal/instance:
08:06:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7841f31884c4209acb421eb987aa665"
08:06:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:05 on dist-test-slave-r4nz"
08:06:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.664510 14061 fs_manager.cc:696] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
08:06:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.669164 14077 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:05.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.669999 14061 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
08:06:05.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.670307 14061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal
08:06:05.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b7841f31884c4209acb421eb987aa665"
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:05 on dist-test-slave-r4nz"
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.670640 14061 fs_report.cc:389] FS layout report
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data/data
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.722077 14061 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:05.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.723546 14061 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:05.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.723973 14061 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.787240 14061 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.126:40703
08:06:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.787299 14128 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.126:40703 every 8 connection(s)
08:06:05.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.789754 14061 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/data/info.pb
08:06:05.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.794163 14129 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.797391 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14061
08:06:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.797767 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-0/wal/instance
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.800796 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:05.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/logs
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data/info.pb
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.125:41845
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.125
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:05.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json with env {}
08:06:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.803547 14129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
08:06:05.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.828783 14129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } has no permanent_uuid. Determining permanent_uuid...
08:06:05.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.831053 14079 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.835086 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.884631 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 1
08:06:05.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.888329 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:05.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:05.956811 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 2
08:06:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:05.961589 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.053014 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 3
08:06:06.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.056521 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:06.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.083475 14131 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.083945 14131 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.084335 14131 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.110953 14131 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.111203 14131 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.111377 14131 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.111542 14131 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.111743 14131 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:06.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.111871 14131 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41883/jwks.json
08:06:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.142215 14131 master_runner.cc:386] Master server non-default flags:
08:06:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.125:41845
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data/info.pb
08:06:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.125
08:06:06.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:06.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/logs
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.143440 14131 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:06.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.144775 14131 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.155329 14140 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:06.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.199975 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 4
08:06:06.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.207362 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:06.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:06.506856 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 5
08:06:06.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.523857 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:07.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.071350 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 6
08:06:07.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.096802 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.125:41845: Network error: Client connection negotiation failed: client connection to 127.13.184.125:41845: connect: Connection refused (error 111)
08:06:07.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:06.155665 14141 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:07.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.389655 14142 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1232 milliseconds
08:06:07.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.391018 14143 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.394057 14131 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.239s user 0.429s sys 0.796s
08:06:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.394407 14131 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.239s user 0.433s sys 0.796s
08:06:07.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.394577 14131 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:07.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.396806 14131 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:07.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.399469 14131 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:07.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.400797 14131 hybrid_clock.cc:648] HybridClock initialized: now 1756800367400753 us; error 52 us; skew 500 ppm
08:06:07.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.401660 14131 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.408394 14131 webserver.cc:480] Webserver started at http://127.13.184.125:34105/ using document root <none> and password file <none>
08:06:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.409245 14131 fs_manager.cc:362] Metadata directory not provided
08:06:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.409456 14131 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:07.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.409929 14131 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:07.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.413908 14131 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data/instance:
08:06:07.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "248f1ca1513c4079ae1182a4ae4afa0b"
08:06:07.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:07 on dist-test-slave-r4nz"
08:06:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.415014 14131 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal/instance:
08:06:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "248f1ca1513c4079ae1182a4ae4afa0b"
08:06:07.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:07 on dist-test-slave-r4nz"
08:06:07.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.422387 14131 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.003s
08:06:07.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.428033 14152 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.429126 14131 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
08:06:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.429440 14131 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal
08:06:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "248f1ca1513c4079ae1182a4ae4afa0b"
08:06:07.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:07 on dist-test-slave-r4nz"
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.429742 14131 fs_report.cc:389] FS layout report
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data/data
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:07.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:07.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.487252 14131 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.488569 14131 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.489004 14131 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:07.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.552132 14131 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.125:41845
08:06:07.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.552207 14203 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.125:41845 every 8 connection(s)
08:06:07.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.554541 14131 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/data/info.pb
08:06:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.558732 14204 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.562804 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14131
08:06:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.563222 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-1/wal/instance
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.565771 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/logs
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data/info.pb
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.124:37511
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.124
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json with env {}
08:06:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.571354 14204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
08:06:07.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.597922 14204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } has no permanent_uuid. Determining permanent_uuid...
08:06:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.610323 14204 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } has no permanent_uuid. Determining permanent_uuid...
08:06:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.612339 14155 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.616252 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.665776 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 1
08:06:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.668850 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:07.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.737284 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 2
08:06:07.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.740574 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.831670 14206 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.831964 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 3
08:06:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.832224 14206 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.832746 14206 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.835407 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.859717 14206 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.860010 14206 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.860337 14206 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.860625 14206 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:07.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.860879 14206 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:07.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.861105 14206 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41883/jwks.json
08:06:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.896533 14206 master_runner.cc:386] Master server non-default flags:
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.124:37511
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41883/jwks.json
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data/info.pb
08:06:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.124
08:06:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/logs
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.897981 14206 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.899945 14206 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.908757 14216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:07.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:07.978744 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 4
08:06:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.984947 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.157279 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } attempt: 7
08:06:08.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.202669 14129 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } has no permanent_uuid. Determining permanent_uuid...
08:06:08.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.213059 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.271536 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 1
08:06:08.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.284389 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 5
08:06:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.284521 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.290979 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.365881 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 2
08:06:08.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.374300 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.459779 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 3
08:06:08.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.466437 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.606884 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 4
08:06:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.613265 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.838423 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 6
08:06:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.864411 14204 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:08.896744 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 5
08:06:08.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:08.903772 14129 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.184.124:37511: Network error: Client connection negotiation failed: client connection to 127.13.184.124:37511: connect: Connection refused (error 111)
08:06:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:07.910108 14217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.198421 14219 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.199635 14218 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1285 milliseconds
08:06:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.200214 14206 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.291s user 0.501s sys 0.778s
08:06:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.200537 14206 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.291s user 0.501s sys 0.778s
08:06:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.200798 14206 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.202224 14206 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:09.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.205251 14206 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:09.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.206852 14206 hybrid_clock.cc:648] HybridClock initialized: now 1756800369206812 us; error 33 us; skew 500 ppm
08:06:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.207877 14206 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:09.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.216042 14206 webserver.cc:480] Webserver started at http://127.13.184.124:35549/ using document root <none> and password file <none>
08:06:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.217216 14206 fs_manager.cc:362] Metadata directory not provided
08:06:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.217492 14206 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.218005 14206 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.224308 14206 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data/instance:
08:06:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71de267f15b445849fc3855bad3e0d19"
08:06:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:09 on dist-test-slave-r4nz"
08:06:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.225683 14206 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal/instance:
08:06:09.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71de267f15b445849fc3855bad3e0d19"
08:06:09.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:09 on dist-test-slave-r4nz"
08:06:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.234297 14206 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
08:06:09.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.239624 14232 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.241533 14206 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.241783 14206 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "71de267f15b445849fc3855bad3e0d19"
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:09 on dist-test-slave-r4nz"
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.242017 14206 fs_report.cc:389] FS layout report
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data/data
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:09.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.307192 14206 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:09.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.308413 14206 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:09.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.308784 14206 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.372167 14206 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.124:37511
08:06:09.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.372208 14283 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.124:37511 every 8 connection(s)
08:06:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.374646 14206 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/data/info.pb
08:06:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.376513 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14206
08:06:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.376963 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/master-2/wal/instance
08:06:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.379308 14284 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.394613 14284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
08:06:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.425442 14284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } has no permanent_uuid. Determining permanent_uuid...
08:06:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.433440 14284 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } has no permanent_uuid. Determining permanent_uuid...
08:06:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.445215 14284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: Bootstrap starting.
08:06:09.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.451861 14284 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: Neither blocks nor log segments found. Creating new log.
08:06:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.453773 14284 log.cc:826] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: Log is configured to *not* fsync() on all Append() calls
08:06:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.456248 14129 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 6
08:06:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.458329 14284 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: No bootstrap required, opened a new log
08:06:09.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.471180 14129 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: Bootstrap starting.
08:06:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.476224 14284 raft_consensus.cc:357] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.476744 14284 raft_consensus.cc:383] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.476899 14284 raft_consensus.cc:738] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 71de267f15b445849fc3855bad3e0d19, State: Initialized, Role: FOLLOWER
08:06:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.477049 14129 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: Neither blocks nor log segments found. Creating new log.
08:06:09.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.477454 14284 consensus_queue.cc:260] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.478993 14129 log.cc:826] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: Log is configured to *not* fsync() on all Append() calls
08:06:09.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.479746 14294 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.480530 14294 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: This master's current role is: FOLLOWER
08:06:09.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.481407 14284 sys_catalog.cc:564] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: configured and running, proceeding with master startup.
08:06:09.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.483631 14129 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: No bootstrap required, opened a new log
08:06:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.492774 14294 raft_consensus.cc:491] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:06:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.493199 14294 raft_consensus.cc:513] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.495296 14294 leader_election.cc:290] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b7841f31884c4209acb421eb987aa665 (127.13.184.126:40703), 248f1ca1513c4079ae1182a4ae4afa0b (127.13.184.125:41845)
08:06:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.511261 14129 raft_consensus.cc:357] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.512317 14129 raft_consensus.cc:383] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.512619 14129 raft_consensus.cc:738] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b7841f31884c4209acb421eb987aa665, State: Initialized, Role: FOLLOWER
08:06:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.513293 14306 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:06:09.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.513552 14306 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:06:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.513401 14129 consensus_queue.cc:260] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.516327 14308 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.517421 14308 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: This master's current role is: FOLLOWER
08:06:09.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.518430 14129 sys_catalog.cc:564] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: configured and running, proceeding with master startup.
08:06:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.518568 14104 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "71de267f15b445849fc3855bad3e0d19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b7841f31884c4209acb421eb987aa665" is_pre_election: true
08:06:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.519304 14104 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 71de267f15b445849fc3855bad3e0d19 in term 0.
08:06:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.520421 14236 leader_election.cc:304] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 71de267f15b445849fc3855bad3e0d19, b7841f31884c4209acb421eb987aa665; no voters:
08:06:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.521164 14294 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:06:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.521476 14294 raft_consensus.cc:491] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:06:09.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.521761 14294 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 0 FOLLOWER]: Advancing to term 1
08:06:09.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.530225 14294 raft_consensus.cc:513] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.533555 14104 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "71de267f15b445849fc3855bad3e0d19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b7841f31884c4209acb421eb987aa665"
08:06:09.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.533999 14104 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 0 FOLLOWER]: Advancing to term 1
08:06:09.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.540544 14104 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 71de267f15b445849fc3855bad3e0d19 in term 1.
08:06:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.542673 14294 leader_election.cc:290] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [CANDIDATE]: Term 1 election: Requested vote from peers b7841f31884c4209acb421eb987aa665 (127.13.184.126:40703), 248f1ca1513c4079ae1182a4ae4afa0b (127.13.184.125:41845)
08:06:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.545909 14236 leader_election.cc:304] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 71de267f15b445849fc3855bad3e0d19, b7841f31884c4209acb421eb987aa665; no voters:
08:06:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.546598 14294 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 1 FOLLOWER]: Leader election won for term 1
08:06:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.551112 14294 raft_consensus.cc:695] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [term 1 LEADER]: Becoming Leader. State: Replica: 71de267f15b445849fc3855bad3e0d19, State: Running, Role: LEADER
08:06:09.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.551918 14294 consensus_queue.cc:237] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:09.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.506189 14206 master_runner.cc:186] Error getting master registration for 127.13.184.126:40703: OK, instance_id { permanent_uuid: "b7841f31884c4209acb421eb987aa665" instance_seqno: 1756800365771116 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:06:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.559672 14206 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.563124 14238 tablet.cc:2378] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:06:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.577503 14311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 71de267f15b445849fc3855bad3e0d19. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.579072 14311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: This master's current role is: LEADER
08:06:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.583945 14315 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:06:09.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.597647 14315 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:06:09.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.624256 14104 raft_consensus.cc:1273] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [term 1 FOLLOWER]: Refusing update from remote peer 71de267f15b445849fc3855bad3e0d19: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:06:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.625413 14311 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:06:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.645483 14308 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 71de267f15b445849fc3855bad3e0d19. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.646111 14308 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: This master's current role is: FOLLOWER
08:06:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.658767 14311 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.659425 14311 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: This master's current role is: LEADER
08:06:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.665313 14325 catalog_manager.cc:1560] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:06:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.668411 14315 catalog_manager.cc:1349] Generated new cluster ID: f67b9a7f45d84df18b27707fc7f1f5a6
08:06:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.668653 14315 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:06:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:09.668927 14325 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:06:09.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.683979 14308 sys_catalog.cc:455] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:09.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.684653 14308 sys_catalog.cc:458] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665 [sys.catalog]: This master's current role is: FOLLOWER
08:06:09.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.703962 14315 catalog_manager.cc:1372] Generated new certificate authority record
08:06:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.705796 14315 catalog_manager.cc:1506] Loading token signing keys...
08:06:09.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.686237 14061 master_runner.cc:186] Error getting master registration for 127.13.184.125:41845: OK, instance_id { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" instance_seqno: 1756800367536215 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:06:09.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.742611 14061 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:09.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.746410 14315 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19: Generated new TSK 0
08:06:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.747311 14315 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:06:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.782231 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal
08:06:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/logs
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data/info.pb
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.65:0
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.65
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.65
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:06:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.924894 14204 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } attempt: 7
08:06:09.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.974933 14204 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: Bootstrap starting.
08:06:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.992092 14204 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: Neither blocks nor log segments found. Creating new log.
08:06:09.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:09.997912 14204 log.cc:826] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: Log is configured to *not* fsync() on all Append() calls
08:06:10.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.018049 14204 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: No bootstrap required, opened a new log
08:06:10.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.077859 14204 raft_consensus.cc:357] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:10.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.079990 14204 raft_consensus.cc:383] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.081612 14204 raft_consensus.cc:738] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 248f1ca1513c4079ae1182a4ae4afa0b, State: Initialized, Role: FOLLOWER
08:06:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.084436 14204 consensus_queue.cc:260] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } }
08:06:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.097467 14332 sys_catalog.cc:455] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.098155 14332 sys_catalog.cc:458] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: This master's current role is: FOLLOWER
08:06:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.104635 14177 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [term 0 FOLLOWER]: Advancing to term 1
08:06:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.110370 14178 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "71de267f15b445849fc3855bad3e0d19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "248f1ca1513c4079ae1182a4ae4afa0b"
08:06:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.111810 14179 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "71de267f15b445849fc3855bad3e0d19" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" is_pre_election: true
08:06:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.117321 14177 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [term 1 FOLLOWER]: Refusing update from remote peer 71de267f15b445849fc3855bad3e0d19: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:06:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.118942 14294 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [LEADER]: Connected to new peer: Peer: permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:06:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.127303 14204 sys_catalog.cc:564] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: configured and running, proceeding with master startup.
08:06:10.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.188789 14332 sys_catalog.cc:455] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 71de267f15b445849fc3855bad3e0d19. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:10.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.189532 14332 sys_catalog.cc:458] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: This master's current role is: FOLLOWER
08:06:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.190675 14338 sys_catalog.cc:455] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:10.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.191383 14338 sys_catalog.cc:458] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b [sys.catalog]: This master's current role is: FOLLOWER
08:06:10.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.243333 14351 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:06:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.278671 14329 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.279134 14329 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:10.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.279637 14329 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.308596 14329 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:10.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.309460 14329 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.184.65
08:06:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.340517 14329 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal
08:06:10.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.65:0
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data/info.pb
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.65
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.65
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/logs
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.342034 14329 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.344046 14329 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:10.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.356454 14354 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.675379 14325 catalog_manager.cc:1261] Loaded cluster ID: f67b9a7f45d84df18b27707fc7f1f5a6
08:06:10.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.676214 14325 catalog_manager.cc:1554] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: loading cluster ID for follower catalog manager: success
08:06:10.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.684494 14325 catalog_manager.cc:1576] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: acquiring CA information for follower catalog manager: success
08:06:10.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.697504 14325 catalog_manager.cc:1604] T 00000000000000000000000000000000 P b7841f31884c4209acb421eb987aa665: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.753311 14294 sys_catalog.cc:455] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "71de267f15b445849fc3855bad3e0d19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b7841f31884c4209acb421eb987aa665" member_type: VOTER last_known_addr { host: "127.13.184.126" port: 40703 } } peers { permanent_uuid: "248f1ca1513c4079ae1182a4ae4afa0b" member_type: VOTER last_known_addr { host: "127.13.184.125" port: 41845 } } peers { permanent_uuid: "71de267f15b445849fc3855bad3e0d19" member_type: VOTER last_known_addr { host: "127.13.184.124" port: 37511 } } }
08:06:10.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:10.754235 14294 sys_catalog.cc:458] T 00000000000000000000000000000000 P 71de267f15b445849fc3855bad3e0d19 [sys.catalog]: This master's current role is: LEADER
08:06:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.266893 14351 catalog_manager.cc:1261] Loaded cluster ID: f67b9a7f45d84df18b27707fc7f1f5a6
08:06:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.267259 14351 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: loading cluster ID for follower catalog manager: success
08:06:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.295467 14351 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: acquiring CA information for follower catalog manager: success
08:06:11.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.329792 14351 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 248f1ca1513c4079ae1182a4ae4afa0b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:11.756780 14353 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 14329
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:11.925046 14353 kernel_stack_watchdog.cc:198] Thread 14329 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:11.927037 14329 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.569s user 0.287s sys 0.665s
08:06:11.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:11.928184 14329 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.570s user 0.288s sys 0.666s
08:06:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:10.359512 14355 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:11.936486 14364 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.951987 14329 server_base.cc:1047] running on GCE node
08:06:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.953392 14329 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.962606 14329 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:11.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.964205 14329 hybrid_clock.cc:648] HybridClock initialized: now 1756800371964171 us; error 35 us; skew 500 ppm
08:06:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.965180 14329 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.990442 14329 webserver.cc:480] Webserver started at http://127.13.184.65:41503/ using document root <none> and password file <none>
08:06:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.991537 14329 fs_manager.cc:362] Metadata directory not provided
08:06:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.991781 14329 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:11.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.992293 14329 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.997772 14329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data/instance:
08:06:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c699348af121449bb1d539a5294449ad"
08:06:11.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:11 on dist-test-slave-r4nz"
08:06:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:11.999768 14329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal/instance:
08:06:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c699348af121449bb1d539a5294449ad"
08:06:12.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:11 on dist-test-slave-r4nz"
08:06:12.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.014812 14329 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.004s sys 0.008s
08:06:12.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.032431 14369 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:12.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.033706 14329 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.002s sys 0.010s
08:06:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.034184 14329 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal
08:06:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c699348af121449bb1d539a5294449ad"
08:06:12.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:11 on dist-test-slave-r4nz"
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.034782 14329 fs_report.cc:389] FS layout report
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data/data
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:12.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:12.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.116461 14329 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.118094 14329 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:12.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.118568 14329 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.121660 14329 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.125824 14329 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.126008 14329 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.126299 14329 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.126467 14329 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:06:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.280404 14329 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.65:40409
08:06:12.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.280536 14481 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.65:40409 every 8 connection(s)
08:06:12.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.282836 14329 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/data/info.pb
08:06:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.291641 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14329
08:06:12.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.292016 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-0/wal/instance
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.305809 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/logs
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data/info.pb
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:12.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.66:0
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.66
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.66
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:12.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:06:12.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.329566 14484 heartbeater.cc:344] Connected to a master server at 127.13.184.126:40703
08:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.330013 14484 heartbeater.cc:461] Registering TS with master...
08:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.330049 14483 heartbeater.cc:344] Connected to a master server at 127.13.184.125:41845
08:06:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.330370 14483 heartbeater.cc:461] Registering TS with master...
08:06:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.331280 14483 heartbeater.cc:507] Master 127.13.184.125:41845 requested a full tablet report, sending...
08:06:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.331259 14484 heartbeater.cc:507] Master 127.13.184.126:40703 requested a full tablet report, sending...
08:06:12.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.334406 14169 ts_manager.cc:194] Registered new tserver with Master: c699348af121449bb1d539a5294449ad (127.13.184.65:40409)
08:06:12.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.334403 14093 ts_manager.cc:194] Registered new tserver with Master: c699348af121449bb1d539a5294449ad (127.13.184.65:40409)
08:06:12.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.340205 14482 heartbeater.cc:344] Connected to a master server at 127.13.184.124:37511
08:06:12.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.340451 14482 heartbeater.cc:461] Registering TS with master...
08:06:12.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.341049 14482 heartbeater.cc:507] Master 127.13.184.124:37511 requested a full tablet report, sending...
08:06:12.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.342757 14249 ts_manager.cc:194] Registered new tserver with Master: c699348af121449bb1d539a5294449ad (127.13.184.65:40409)
08:06:12.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.344383 14249 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.184.65:49687
08:06:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.598600 14492 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.599071 14492 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.599584 14492 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:12.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.625277 14492 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.626026 14492 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.184.66
08:06:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.653728 14492 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.66:0
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data/info.pb
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.66
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:12.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.66
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/logs
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.655030 14492 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:12.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:12.656759 14492 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:12.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.667850 14498 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:13.347060 14482 heartbeater.cc:499] Master 127.13.184.124:37511 was elected leader, sending a full tablet report...
08:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:12.670518 14499 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.071122 14497 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 14492
08:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.603524 14497 kernel_stack_watchdog.cc:198] Thread 14492 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:06:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.604245 14492 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.936s user 0.634s sys 1.218s
08:06:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.604684 14492 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.937s user 0.635s sys 1.218s
08:06:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.605463 14501 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:14.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:14.607705 14500 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1935 milliseconds
08:06:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.607722 14492 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.608784 14492 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:14.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.610605 14492 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.611922 14492 hybrid_clock.cc:648] HybridClock initialized: now 1756800374611865 us; error 38 us; skew 500 ppm
08:06:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.612579 14492 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.617887 14492 webserver.cc:480] Webserver started at http://127.13.184.66:35691/ using document root <none> and password file <none>
08:06:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.618731 14492 fs_manager.cc:362] Metadata directory not provided
08:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.618898 14492 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:14.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.619235 14492 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.623126 14492 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data/instance:
08:06:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fbdace2a4b4487a99f54b702ce00738"
08:06:14.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:14 on dist-test-slave-r4nz"
08:06:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.624043 14492 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal/instance:
08:06:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fbdace2a4b4487a99f54b702ce00738"
08:06:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:14 on dist-test-slave-r4nz"
08:06:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.630089 14492 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
08:06:14.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.634733 14510 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.635558 14492 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.635779 14492 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9fbdace2a4b4487a99f54b702ce00738"
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:14 on dist-test-slave-r4nz"
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.636008 14492 fs_report.cc:389] FS layout report
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data/data
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:14.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:14.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.674621 14492 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.675876 14492 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.676223 14492 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:14.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.678283 14492 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.681687 14492 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.681846 14492 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:14.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.682029 14492 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:14.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.682197 14492 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
08:06:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.809921 14492 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.66:40415
08:06:14.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.810016 14622 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.66:40415 every 8 connection(s)
08:06:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.812194 14492 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/data/info.pb
08:06:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.821233 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14492
08:06:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.821903 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-1/wal/instance
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.835955 14049 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/logs
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data/info.pb
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.67:0
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.67
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.67
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:14.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:06:14.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.857465 14624 heartbeater.cc:344] Connected to a master server at 127.13.184.125:41845
08:06:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.857874 14624 heartbeater.cc:461] Registering TS with master...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.858683 14625 heartbeater.cc:344] Connected to a master server at 127.13.184.126:40703
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.858996 14625 heartbeater.cc:461] Registering TS with master...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.859053 14624 heartbeater.cc:507] Master 127.13.184.125:41845 requested a full tablet report, sending...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.859755 14625 heartbeater.cc:507] Master 127.13.184.126:40703 requested a full tablet report, sending...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.860255 14623 heartbeater.cc:344] Connected to a master server at 127.13.184.124:37511
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.860550 14623 heartbeater.cc:461] Registering TS with master...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.861304 14623 heartbeater.cc:507] Master 127.13.184.124:37511 requested a full tablet report, sending...
08:06:14.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.861912 14169 ts_manager.cc:194] Registered new tserver with Master: 9fbdace2a4b4487a99f54b702ce00738 (127.13.184.66:40415)
08:06:14.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.863216 14249 ts_manager.cc:194] Registered new tserver with Master: 9fbdace2a4b4487a99f54b702ce00738 (127.13.184.66:40415)
08:06:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.864703 14249 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.184.66:53539
08:06:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:14.866573 14093 ts_manager.cc:194] Registered new tserver with Master: 9fbdace2a4b4487a99f54b702ce00738 (127.13.184.66:40415)
08:06:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.104728 14633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.105089 14633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.105458 14633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.134605 14633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:15.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.135313 14633 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.184.67
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:15.164011 14633 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.184.84:41673
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.184.67:0
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data/info.pb
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.184.67
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.184.126:40703,127.13.184.125:41845,127.13.184.124:37511
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.184.67
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/logs
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:15.165266 14633 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:15.166692 14633 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:15.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.176290 14639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:15.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:15.867879 14623 heartbeater.cc:499] Master 127.13.184.124:37511 was elected leader, sending a full tablet report...
08:06:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.178001 14640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:15.181471 14642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:16.258728 14641 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1077 milliseconds
08:06:16.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.258850 14633 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:16.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.259845 14633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:16.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.261844 14633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:16.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.263159 14633 hybrid_clock.cc:648] HybridClock initialized: now 1756800376263097 us; error 77 us; skew 500 ppm
08:06:16.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.263909 14633 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.269829 14633 webserver.cc:480] Webserver started at http://127.13.184.67:34207/ using document root <none> and password file <none>
08:06:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.270713 14633 fs_manager.cc:362] Metadata directory not provided
08:06:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.270918 14633 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.271380 14633 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.276177 14633 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data/instance:
08:06:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b163658e7d4724bf2a779cb07a5dd8"
08:06:16.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:16 on dist-test-slave-r4nz"
08:06:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.277148 14633 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal/instance:
08:06:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b163658e7d4724bf2a779cb07a5dd8"
08:06:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:16 on dist-test-slave-r4nz"
08:06:16.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.283690 14633 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
08:06:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.288846 14650 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.289685 14633 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.289964 14633 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "48b163658e7d4724bf2a779cb07a5dd8"
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:16 on dist-test-slave-r4nz"
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.290278 14633 fs_report.cc:389] FS layout report
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal
08:06:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data/data
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:16.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.332561 14633 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:16.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.333797 14633 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:16.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.334218 14633 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.337224 14633 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.340865 14633 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.341055 14633 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.341320 14633 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.341478 14633 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:06:16.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.477322 14633 rpc_server.cc:307] RPC server started. Bound to: 127.13.184.67:39773
08:06:16.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.477378 14764 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.184.67:39773 every 8 connection(s)
08:06:16.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.479636 14633 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/data/info.pb
08:06:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.480952 14049 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14633
08:06:16.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.481458 14049 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster5072587665352384195/ts-2/wal/instance
08:06:16.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.529279 14766 heartbeater.cc:344] Connected to a master server at 127.13.184.125:41845
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.529774 14766 heartbeater.cc:461] Registering TS with master...
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.530226 14765 heartbeater.cc:344] Connected to a master server at 127.13.184.124:37511
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.530294 14767 heartbeater.cc:344] Connected to a master server at 127.13.184.126:40703
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.530557 14765 heartbeater.cc:461] Registering TS with master...
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.530654 14767 heartbeater.cc:461] Registering TS with master...
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.531088 14766 heartbeater.cc:507] Master 127.13.184.125:41845 requested a full tablet report, sending...
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.531389 14765 heartbeater.cc:507] Master 127.13.184.124:37511 requested a full tablet report, sending...
08:06:16.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.531527 14767 heartbeater.cc:507] Master 127.13.184.126:40703 requested a full tablet report, sending...
08:06:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.533545 14249 ts_manager.cc:194] Registered new tserver with Master: 48b163658e7d4724bf2a779cb07a5dd8 (127.13.184.67:39773)
08:06:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.534278 14168 ts_manager.cc:194] Registered new tserver with Master: 48b163658e7d4724bf2a779cb07a5dd8 (127.13.184.67:39773)
08:06:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.535075 14093 ts_manager.cc:194] Registered new tserver with Master: 48b163658e7d4724bf2a779cb07a5dd8 (127.13.184.67:39773)
08:06:16.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.535246 14249 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.184.67:55281
08:06:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:16.552829 14049 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:06:16.563 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:06:16.565 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:06:16.581 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.184.126"
port: 40703
}
bound_http_address {
host: "127.13.184.126"
port: 45679
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.184.125"
port: 41845
}
bound_http_address {
host: "127.13.184.125"
port: 34105
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.184.124"
port: 37511
}
bound_http_address {
host: "127.13.184.124"
port: 35549
}
}
}
08:06:16.585 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:06:16.587 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.13.184.65"
port: 40409
}
bound_http_address {
host: "127.13.184.65"
port: 41503
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.13.184.66"
port: 40415
}
bound_http_address {
host: "127.13.184.66"
port: 35691
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.13.184.67"
port: 39773
}
bound_http_address {
host: "127.13.184.67"
port: 34207
}
}
}
08:06:16.588 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:06:16.741 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:06:16.762 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:06:16.873 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskA19hJ6/test-tmp
08:06:16.874 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:06:16.874 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskA19hJ6/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:06:16.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:06:17.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:17.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:06:17.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:17.162182 14776 mini_oidc.cc:150] Starting JWKS server
08:06:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:17.165611 14776 webserver.cc:480] Webserver started at http://127.0.0.1:44201/ using document root <none> and password file <none>
08:06:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:17.165979 14776 mini_oidc.cc:186] Starting OIDC Discovery server
08:06:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:17.167702 14776 webserver.cc:480] Webserver started at http://127.0.0.1:43417/ using document root <none> and password file <none>
08:06:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:06:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/principal' for realm 'KRBTEST.COM',
08:06:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:06:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:17 dist-test-slave-r4nz krb5kdc[14788](info): setting up network...
08:06:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:06:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:17 dist-test-slave-r4nz krb5kdc[14788](info): set up 2 sockets
08:06:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:17 dist-test-slave-r4nz krb5kdc[14788](info): commencing operation
08:06:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:06:17.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:17.538544 14765 heartbeater.cc:499] Master 127.13.184.124:37511 was elected leader, sending a full tablet report...
08:06:19.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.048426 14776 mini_kdc.cc:121] Time spent starting KDC: real 1.849s user 0.000s sys 0.015s
08:06:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:06:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:06:19.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:06:19.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:06:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:06:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:06:19.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:06:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:06:19.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:06:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:06:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/test-user.keytab.
08:06:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/test-user.keytab.
08:06:19.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:19 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:19.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:06:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:19Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:06:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:19Z Disabled control of system clock
08:06:19.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.62@KRBTEST.COM; defaulting to no policy
08:06:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.62@KRBTEST.COM" created.
08:06:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:19.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.62@KRBTEST.COM; defaulting to no policy
08:06:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.62@KRBTEST.COM" created.
08:06:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:19.233397 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:19.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/logs
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:19.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.62:34081
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.62
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.62
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:19.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:19.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.402949 14619 debug-util.cc:398] Leaking SignalData structure 0x7b08000b87e0 after lost signal to thread 14493
08:06:19.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.403658 14619 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8e60 after lost signal to thread 14622
08:06:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.506737 14805 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:19.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.507304 14805 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:19.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.507927 14805 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.534318 14805 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.534546 14805 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.534916 14805 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.535188 14805 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.535492 14805 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.535740 14805 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.535924 14805 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:19.584698 14805 master_runner.cc:386] Master server non-default flags:
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.62
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.62:34081
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.62
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/logs
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:19.586109 14805 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:19.587333 14805 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:19.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.598347 14811 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.602555 14814 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:19.598721 14812 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:20.946038 14813 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1345 milliseconds
08:06:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.946193 14805 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:20.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.947821 14805 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:20.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.950868 14805 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:20.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.952447 14805 hybrid_clock.cc:648] HybridClock initialized: now 1756800380952397 us; error 38 us; skew 500 ppm
08:06:20.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.953539 14805 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:20 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800380, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:20.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.960242 14805 init.cc:377] Logged in from keytab as kudu/127.14.110.62@KRBTEST.COM (short username kudu)
08:06:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.965137 14805 webserver.cc:480] Webserver started at http://127.14.110.62:32877/ using document root <none> and password file <none>
08:06:20.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.965919 14805 fs_manager.cc:362] Metadata directory not provided
08:06:20.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.966081 14805 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:20.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.966503 14805 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.970353 14805 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/instance:
08:06:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:20.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:20 on dist-test-slave-r4nz"
08:06:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.971343 14805 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal/instance:
08:06:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:20.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:20 on dist-test-slave-r4nz"
08:06:20.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.978024 14805 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
08:06:20.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.982836 14823 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.984464 14805 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:06:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.984701 14805 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:20.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:20 on dist-test-slave-r4nz"
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:20.984923 14805 fs_report.cc:389] FS layout report
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/data
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:20.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:21.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.047489 14805 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.055051 14805 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.055401 14805 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.136217 14805 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.62:34081
08:06:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.136343 14874 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.62:34081 every 8 connection(s)
08:06:21.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.138676 14805 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.144404 14875 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.146857 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14805
08:06:21.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.147258 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal/instance
08:06:21.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.61@KRBTEST.COM; defaulting to no policy
08:06:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.159581 14875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } has no permanent_uuid. Determining permanent_uuid...
08:06:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:21.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.61@KRBTEST.COM" created.
08:06:21.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:21 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800380, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.203740 14875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } has no permanent_uuid. Determining permanent_uuid...
08:06:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.61@KRBTEST.COM; defaulting to no policy
08:06:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.61@KRBTEST.COM" created.
08:06:21.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.206065 14824 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.210386 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:21.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.223776 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/logs
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.61:45735
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.61
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.61
08:06:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:21.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.259892 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 1
08:06:21.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.263846 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.332202 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 2
08:06:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.336410 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.427799 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 3
08:06:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.431612 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.495757 14884 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.496274 14884 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:21.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.496886 14884 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.524076 14884 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.524256 14884 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.524505 14884 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.524730 14884 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.524915 14884 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.525071 14884 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:21.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.525205 14884 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.559602 14884 master_runner.cc:386] Master server non-default flags:
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.61
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.61:45735
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.61
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:21.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/logs
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.561030 14884 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.562083 14884 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.572574 14890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.574931 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 4
08:06:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.580580 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:21.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:21.880061 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 5
08:06:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.890209 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.437628 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 6
08:06:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:22.469147 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.61:45735: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:21.574563 14891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:22.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:22.931216 14893 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:22.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:22.932114 14892 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1356 milliseconds
08:06:22.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:22.933701 14884 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.361s user 0.450s sys 0.846s
08:06:22.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:22.934041 14884 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.361s user 0.450s sys 0.846s
08:06:22.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.934303 14884 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:22.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.935272 14884 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:22.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.937458 14884 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:22.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.938753 14884 hybrid_clock.cc:648] HybridClock initialized: now 1756800382938702 us; error 62 us; skew 500 ppm
08:06:22.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.939520 14884 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:22.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:22 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800382, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.945209 14884 init.cc:377] Logged in from keytab as kudu/127.14.110.61@KRBTEST.COM (short username kudu)
08:06:22.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.950251 14884 webserver.cc:480] Webserver started at http://127.14.110.61:40297/ using document root <none> and password file <none>
08:06:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.950985 14884 fs_manager.cc:362] Metadata directory not provided
08:06:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.951152 14884 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.951476 14884 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.955209 14884 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/instance:
08:06:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da8f4f913e4740e3a3b3770e4f291cbc"
08:06:22.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:22 on dist-test-slave-r4nz"
08:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.956082 14884 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal/instance:
08:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da8f4f913e4740e3a3b3770e4f291cbc"
08:06:22.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:22 on dist-test-slave-r4nz"
08:06:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.962824 14884 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
08:06:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.968101 14904 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.969182 14884 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
08:06:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.969481 14884 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da8f4f913e4740e3a3b3770e4f291cbc"
08:06:22.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:22 on dist-test-slave-r4nz"
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:22.969815 14884 fs_report.cc:389] FS layout report
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/data
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:22.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:23.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.044483 14884 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.048990 14884 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:23.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.049333 14884 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.109721 14884 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.61:45735
08:06:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.109783 14955 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.61:45735 every 8 connection(s)
08:06:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.112097 14884 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.114909 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14884
08:06:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.115509 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal/instance
08:06:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.117491 14956 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.129433 14956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } has no permanent_uuid. Determining permanent_uuid...
08:06:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.60@KRBTEST.COM; defaulting to no policy
08:06:23.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:23.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.60@KRBTEST.COM" created.
08:06:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:23 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800382, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:23.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.162629 14956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } has no permanent_uuid. Determining permanent_uuid...
08:06:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.60@KRBTEST.COM; defaulting to no policy
08:06:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.60@KRBTEST.COM" created.
08:06:23.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:23 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800382, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:23.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.183180 14956 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } has no permanent_uuid. Determining permanent_uuid...
08:06:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.184968 14905 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.187987 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/logs
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:23.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.60:46255
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.60
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.60
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:23.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.188192 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.237625 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 1
08:06:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.240720 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.309087 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 2
08:06:23.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.312249 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.403671 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 3
08:06:23.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.406630 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.447909 14966 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:23.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.448539 14966 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:23.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.449095 14966 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.474406 14966 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.474611 14966 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.474867 14966 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.475052 14966 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.475337 14966 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.475595 14966 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.475777 14966 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.505944 14966 master_runner.cc:386] Master server non-default flags:
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.60
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.60:46255
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:23.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.60
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/logs
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.507263 14966 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.508319 14966 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:23.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.516888 14972 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:23.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.529611 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } attempt: 7
08:06:23.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:23 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800380, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:23.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.549911 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 4
08:06:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.554119 14875 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } has no permanent_uuid. Determining permanent_uuid...
08:06:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.557283 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.567312 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.625691 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 1
08:06:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.631148 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.712630 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 2
08:06:23.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.719316 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.804854 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 3
08:06:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.811785 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.856674 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 5
08:06:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.862020 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:23.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:23.952304 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 4
08:06:23.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.958086 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:24.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.241659 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 5
08:06:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.247099 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.409546 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 6
08:06:24.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.441077 14956 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:23.518412 14973 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:24.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.674165 14975 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.675992 14974 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1154 milliseconds
08:06:24.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.676071 14966 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:24.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.677135 14966 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.679337 14966 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.680663 14966 hybrid_clock.cc:648] HybridClock initialized: now 1756800384680639 us; error 32 us; skew 500 ppm
08:06:24.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.681291 14966 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:24.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800384, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:24.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.686661 14966 init.cc:377] Logged in from keytab as kudu/127.14.110.60@KRBTEST.COM (short username kudu)
08:06:24.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.691478 14966 webserver.cc:480] Webserver started at http://127.14.110.60:37537/ using document root <none> and password file <none>
08:06:24.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.692234 14966 fs_manager.cc:362] Metadata directory not provided
08:06:24.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.692433 14966 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:24.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.692842 14966 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.696883 14966 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/instance:
08:06:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7635d667c7494f08a38d7a749f01d0e7"
08:06:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:24 on dist-test-slave-r4nz"
08:06:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.697980 14966 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal/instance:
08:06:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7635d667c7494f08a38d7a749f01d0e7"
08:06:24.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:24 on dist-test-slave-r4nz"
08:06:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.704285 14966 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
08:06:24.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.708688 14989 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:24.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.709509 14966 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.709813 14966 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7635d667c7494f08a38d7a749f01d0e7"
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:24 on dist-test-slave-r4nz"
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.710091 14966 fs_report.cc:389] FS layout report
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/data
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:24.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:24.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:24.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.756906 14966 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.761387 14966 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.761785 14966 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:24.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.799615 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 6
08:06:24.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.809525 14875 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.832155 14966 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.60:46255
08:06:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.832183 15042 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.60:46255 every 8 connection(s)
08:06:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.834686 14966 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.835918 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 14966
08:06:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.836331 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal/instance
08:06:24.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.840095 15043 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:06:24.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.856866 15043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } has no permanent_uuid. Determining permanent_uuid...
08:06:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:24.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800384, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:24.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.886469 15043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } has no permanent_uuid. Determining permanent_uuid...
08:06:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800384, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:24.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800379, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:24.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.906937 15043 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } has no permanent_uuid. Determining permanent_uuid...
08:06:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:24 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800384, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.927330 15043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Bootstrap starting.
08:06:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.932909 15043 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Neither blocks nor log segments found. Creating new log.
08:06:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.934428 15043 log.cc:826] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Log is configured to *not* fsync() on all Append() calls
08:06:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.938539 15043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: No bootstrap required, opened a new log
08:06:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.953682 15043 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.954209 15043 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.954387 15043 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7635d667c7494f08a38d7a749f01d0e7, State: Initialized, Role: FOLLOWER
08:06:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.954881 15043 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:24.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.957172 15053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.957832 15053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: FOLLOWER
08:06:24.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.958647 15043 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: configured and running, proceeding with master startup.
08:06:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.979977 15064 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:06:24.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:24.982600 15064 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:06:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:24.976203 14966 master_runner.cc:186] Error getting master registration for 127.14.110.62:34081: OK, instance_id { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" instance_seqno: 1756800381114475 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:06:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.004474 14966 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:25.015939 14995 tablet.cc:2378] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:06:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.057402 15053 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:06:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.057909 15053 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.060220 15053 leader_election.cc:290] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081), da8f4f913e4740e3a3b3770e4f291cbc (127.14.110.61:45735)
08:06:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.501544 14956 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 7
08:06:25.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:25 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800382, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.538364 14956 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Bootstrap starting.
08:06:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.544328 14956 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Neither blocks nor log segments found. Creating new log.
08:06:25.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.546303 14956 log.cc:826] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Log is configured to *not* fsync() on all Append() calls
08:06:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.550673 14956 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: No bootstrap required, opened a new log
08:06:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.572455 14956 raft_consensus.cc:357] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.573102 14956 raft_consensus.cc:383] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.573501 14956 raft_consensus.cc:738] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: da8f4f913e4740e3a3b3770e4f291cbc, State: Initialized, Role: FOLLOWER
08:06:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.574290 14956 consensus_queue.cc:260] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:25.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.585325 15070 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.586766 14931 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7635d667c7494f08a38d7a749f01d0e7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" is_pre_election: true
08:06:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.587585 15070 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: FOLLOWER
08:06:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.588049 14956 sys_catalog.cc:564] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: configured and running, proceeding with master startup.
08:06:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.588229 14931 raft_consensus.cc:2466] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7635d667c7494f08a38d7a749f01d0e7 in term 0.
08:06:25.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.589282 14990 leader_election.cc:304] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7635d667c7494f08a38d7a749f01d0e7, da8f4f913e4740e3a3b3770e4f291cbc; no voters:
08:06:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.596298 15071 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:06:25.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.596654 15071 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:06:25.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.596946 15071 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 0 FOLLOWER]: Advancing to term 1
08:06:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.605731 15071 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:25.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.607295 15071 leader_election.cc:290] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [CANDIDATE]: Term 1 election: Requested vote from peers 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081), da8f4f913e4740e3a3b3770e4f291cbc (127.14.110.61:45735)
08:06:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.608426 14931 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7635d667c7494f08a38d7a749f01d0e7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da8f4f913e4740e3a3b3770e4f291cbc"
08:06:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.608840 14931 raft_consensus.cc:3058] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 0 FOLLOWER]: Advancing to term 1
08:06:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.615274 14931 raft_consensus.cc:2466] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7635d667c7494f08a38d7a749f01d0e7 in term 1.
08:06:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.618367 14990 leader_election.cc:304] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7635d667c7494f08a38d7a749f01d0e7, da8f4f913e4740e3a3b3770e4f291cbc; no voters:
08:06:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.618932 15071 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 FOLLOWER]: Leader election won for term 1
08:06:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.621609 15071 raft_consensus.cc:695] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 LEADER]: Becoming Leader. State: Replica: 7635d667c7494f08a38d7a749f01d0e7, State: Running, Role: LEADER
08:06:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.622995 15071 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:25.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.632548 15077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7635d667c7494f08a38d7a749f01d0e7. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:25.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.632973 15077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: LEADER
08:06:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.635056 15082 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:06:25.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.645351 15082 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:06:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.667338 14931 raft_consensus.cc:1273] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Refusing update from remote peer 7635d667c7494f08a38d7a749f01d0e7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:06:25.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.668995 15071 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:06:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:25.675405 15085 catalog_manager.cc:1560] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:06:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:25.675908 15085 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:06:25.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.647853 14884 master_runner.cc:186] Error getting master registration for 127.14.110.62:34081: OK, instance_id { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" instance_seqno: 1756800381114475 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:06:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.686445 14884 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.707515 15070 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7635d667c7494f08a38d7a749f01d0e7. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.708631 15070 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: FOLLOWER
08:06:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.708549 15087 mvcc.cc:204] Tried to move back new op lower bound from 7195854379676987392 to 7195854379521150976. Current Snapshot: MvccSnapshot[applied={T|T < 7195854379676987392}]
08:06:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.721518 15070 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:25.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.722262 15070 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: FOLLOWER
08:06:25.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.725346 15077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:25.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.725945 15077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: LEADER
08:06:25.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.728821 15082 catalog_manager.cc:1349] Generated new cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:25.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.729038 15082 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:06:25.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.762805 15082 catalog_manager.cc:1372] Generated new certificate authority record
08:06:25.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.764672 15082 catalog_manager.cc:1506] Loading token signing keys...
08:06:25.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.787560 15082 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Generated new TSK 0
08:06:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.789304 15082 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:06:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.1@KRBTEST.COM; defaulting to no policy
08:06:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.1@KRBTEST.COM" created.
08:06:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.842878 14875 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } attempt: 7
08:06:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:25.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:25.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.1@KRBTEST.COM; defaulting to no policy
08:06:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:25.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.1@KRBTEST.COM" created.
08:06:25.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:25 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800380, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.915184 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:25.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/logs
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data/info.pb
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.1:0
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.1
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.1
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.1
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.917564 14875 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Bootstrap starting.
08:06:25.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.936386 14875 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Neither blocks nor log segments found. Creating new log.
08:06:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.938745 14875 log.cc:826] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Log is configured to *not* fsync() on all Append() calls
08:06:25.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:25.955263 14875 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: No bootstrap required, opened a new log
08:06:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.003099 14875 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.004592 14875 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:06:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.005169 14875 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1abe593fbd60484eb94b87ce539701e2, State: Initialized, Role: FOLLOWER
08:06:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.008919 14875 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.036903 15100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:26.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.038317 15100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:26.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.039163 15100 raft_consensus.cc:491] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:06:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.041184 15100 raft_consensus.cc:513] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.045248 14931 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1abe593fbd60484eb94b87ce539701e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" is_pre_election: true
08:06:26.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.050454 14850 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7635d667c7494f08a38d7a749f01d0e7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1abe593fbd60484eb94b87ce539701e2" is_pre_election: true
08:06:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.051066 14850 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7635d667c7494f08a38d7a749f01d0e7 in term 0.
08:06:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.052026 14848 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 0 FOLLOWER]: Advancing to term 1
08:06:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.058266 14849 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7635d667c7494f08a38d7a749f01d0e7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.060045 15100 leader_election.cc:290] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers da8f4f913e4740e3a3b3770e4f291cbc (127.14.110.61:45735), 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255)
08:06:26.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.061570 15016 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1abe593fbd60484eb94b87ce539701e2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7635d667c7494f08a38d7a749f01d0e7" is_pre_election: true
08:06:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.063117 14824 leader_election.cc:304] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1abe593fbd60484eb94b87ce539701e2; no voters: 7635d667c7494f08a38d7a749f01d0e7, da8f4f913e4740e3a3b3770e4f291cbc
08:06:26.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.064688 14848 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Refusing update from remote peer 7635d667c7494f08a38d7a749f01d0e7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:06:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.065994 14829 tablet.cc:2378] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:06:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.066109 15077 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:06:26.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.075479 14875 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: configured and running, proceeding with master startup.
08:06:26.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.147275 15099 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:06:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.152230 15106 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:26.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.153596 15106 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.154675 15100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7635d667c7494f08a38d7a749f01d0e7. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:26.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.155356 15100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:26.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.220551 15118 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:06:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.331146 15097 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.331779 15097 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:26.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.332715 15097 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:26.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.384697 15097 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:26.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.386161 15097 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.110.1
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.442734 15097 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.1
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.1:0
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:26.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data/info.pb
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.1
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.1
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/logs
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.444231 15097 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.445417 15097 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.459205 15125 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:26.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.577739 15077 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7635d667c7494f08a38d7a749f01d0e7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:26.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.591894 15077 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: LEADER
08:06:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.681039 15085 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.681699 15085 catalog_manager.cc:1554] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: loading cluster ID for follower catalog manager: success
08:06:26.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.688356 15085 catalog_manager.cc:1576] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: acquiring CA information for follower catalog manager: success
08:06:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:26.696913 15085 catalog_manager.cc:1604] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.234076 15118 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.234350 15118 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: loading cluster ID for follower catalog manager: success
08:06:27.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.243595 15118 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: acquiring CA information for follower catalog manager: success
08:06:27.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.248108 15118 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:27.859724 15124 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 15097
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:27.988801 15124 kernel_stack_watchdog.cc:198] Thread 15097 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:27.989154 15097 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.529s user 0.002s sys 0.000s
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:27.989450 15097 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.530s user 0.002s sys 0.000s
08:06:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:26.460723 15126 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:27.995693 15137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:27.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.998024 15097 server_base.cc:1047] running on GCE node
08:06:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:27.999209 15097 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.001519 15097 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:28.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.002980 15097 hybrid_clock.cc:648] HybridClock initialized: now 1756800388002879 us; error 90 us; skew 500 ppm
08:06:28.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.003917 15097 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:28 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800388, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.010651 15097 init.cc:377] Logged in from keytab as kudu/127.14.110.1@KRBTEST.COM (short username kudu)
08:06:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.016641 15097 webserver.cc:480] Webserver started at http://127.14.110.1:45589/ using document root <none> and password file <none>
08:06:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.017706 15097 fs_manager.cc:362] Metadata directory not provided
08:06:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.017938 15097 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.018438 15097 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.023903 15097 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data/instance:
08:06:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e301365419f41138fb4c260fdd8a1c5"
08:06:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:28 on dist-test-slave-r4nz"
08:06:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.025177 15097 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal/instance:
08:06:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e301365419f41138fb4c260fdd8a1c5"
08:06:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:28 on dist-test-slave-r4nz"
08:06:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.033650 15097 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
08:06:28.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.040072 15144 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.041184 15097 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:06:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.041524 15097 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal
08:06:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7e301365419f41138fb4c260fdd8a1c5"
08:06:28.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:28 on dist-test-slave-r4nz"
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.041880 15097 fs_report.cc:389] FS layout report
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data/data
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:28.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.112825 15097 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:28.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.123180 15097 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.123651 15097 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:28.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.143136 15097 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.154209 15097 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.154450 15097 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.154693 15097 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:28.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.154858 15097 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:06:28.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.474635 15097 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.1:41879
08:06:28.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.474738 15256 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.1:41879 every 8 connection(s)
08:06:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.477638 15097 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/data/info.pb
08:06:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.479597 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15097
08:06:28.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.480329 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-0/wal/instance
08:06:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.2@KRBTEST.COM; defaulting to no policy
08:06:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:28.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.2@KRBTEST.COM" created.
08:06:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:28 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800388, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.1@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:28.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:28 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800388, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.1@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:28 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800388, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.1@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.594197 15257 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:28.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.594655 15257 heartbeater.cc:461] Registering TS with master...
08:06:28.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.595876 15257 heartbeater.cc:507] Master 127.14.110.61:45735 requested a full tablet report, sending...
08:06:28.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.603375 14921 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.610512 15262 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:28.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.610831 15262 heartbeater.cc:461] Registering TS with master...
08:06:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:28.611382 14776 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.14.110.2: real 0.117s user 0.001s sys 0.012s
08:06:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.611733 15262 heartbeater.cc:507] Master 127.14.110.62:34081 requested a full tablet report, sending...
08:06:28.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.614543 14840 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.620275 15258 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:28.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.620640 15258 heartbeater.cc:461] Registering TS with master...
08:06:28.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.621592 15258 heartbeater.cc:507] Master 127.14.110.60:46255 requested a full tablet report, sending...
08:06:28.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.624130 15006 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.626636 15006 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.110.1@KRBTEST.COM'} at 127.14.110.1:33429
08:06:28.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.2@KRBTEST.COM; defaulting to no policy
08:06:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.2@KRBTEST.COM" created.
08:06:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:28.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:28.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:28.650523 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:28.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:28.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/logs
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data/info.pb
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.2:0
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.2
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.2
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.2
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:28.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:28.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:28.998389 15271 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:28.998932 15271 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:28.999918 15271 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:29.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:29.050403 15271 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:29.051834 15271 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.110.2
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:29.110495 15271 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.2
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.2:0
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data/info.pb
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.2
08:06:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.2
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/logs
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:29.111989 15271 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:29.113142 15271 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:29.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:29.159229 15277 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:29.630383 15258 heartbeater.cc:499] Master 127.14.110.60:46255 was elected leader, sending a full tablet report...
08:06:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:29.164119 15278 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:30.419747 15271 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.257s user 0.433s sys 0.744s
08:06:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:30.420004 15271 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.258s user 0.433s sys 0.744s
08:06:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.423034 15271 server_base.cc:1047] running on GCE node
08:06:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:30.423168 15285 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.424188 15271 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.426074 15271 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:30.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.427583 15271 hybrid_clock.cc:648] HybridClock initialized: now 1756800390427546 us; error 29 us; skew 500 ppm
08:06:30.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.428597 15271 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:30 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800390, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:30.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.436585 15271 init.cc:377] Logged in from keytab as kudu/127.14.110.2@KRBTEST.COM (short username kudu)
08:06:30.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.442847 15271 webserver.cc:480] Webserver started at http://127.14.110.2:33523/ using document root <none> and password file <none>
08:06:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.443840 15271 fs_manager.cc:362] Metadata directory not provided
08:06:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.444000 15271 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:30.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.444310 15271 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:30.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.448030 15271 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data/instance:
08:06:30.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "beaa2681b4384ce89b2d29030fa685fc"
08:06:30.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:30 on dist-test-slave-r4nz"
08:06:30.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.448997 15271 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal/instance:
08:06:30.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "beaa2681b4384ce89b2d29030fa685fc"
08:06:30.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:30 on dist-test-slave-r4nz"
08:06:30.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.456269 15271 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
08:06:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.461117 15291 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.462083 15271 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
08:06:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.462404 15271 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal
08:06:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "beaa2681b4384ce89b2d29030fa685fc"
08:06:30.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:30 on dist-test-slave-r4nz"
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.462684 15271 fs_report.cc:389] FS layout report
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data/data
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:30.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.520185 15271 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:30.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.524828 15271 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:30.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.525231 15271 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:30.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.527370 15271 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.530915 15271 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.531083 15271 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.531337 15271 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:30.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.531494 15271 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:06:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.653745 15271 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.2:41493
08:06:30.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.653837 15403 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.2:41493 every 8 connection(s)
08:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.655894 15271 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/data/info.pb
08:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.656339 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15271
08:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.656733 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-1/wal/instance
08:06:30.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:30 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800390, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.2@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:30.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.110.3@KRBTEST.COM; defaulting to no policy
08:06:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:30 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800390, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.2@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:30.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:30.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.110.3@KRBTEST.COM" created.
08:06:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.708894 15404 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:30 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800390, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.2@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:30.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.709343 15404 heartbeater.cc:461] Registering TS with master...
08:06:30.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.710333 15404 heartbeater.cc:507] Master 127.14.110.61:45735 requested a full tablet report, sending...
08:06:30.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.712832 14921 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.716180 15405 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.716477 15405 heartbeater.cc:461] Registering TS with master...
08:06:30.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.717197 15405 heartbeater.cc:507] Master 127.14.110.60:46255 requested a full tablet report, sending...
08:06:30.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.718784 15006 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:30.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.720311 15006 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.110.2@KRBTEST.COM'} at 127.14.110.2:54897
08:06:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.722606 15406 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.722899 15406 heartbeater.cc:461] Registering TS with master...
08:06:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.723449 15406 heartbeater.cc:507] Master 127.14.110.62:34081 requested a full tablet report, sending...
08:06:30.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.725576 14840 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:30.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.110.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:30.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.110.3@KRBTEST.COM; defaulting to no policy
08:06:30.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:30.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.110.3@KRBTEST.COM" created.
08:06:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:06:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.110.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab.
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.754737 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/logs
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data/info.pb
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.3:0
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.3
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.3
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:30.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.3
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:30 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800390, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:30.964186 14819 init.cc:295] Successfully reacquired a new kerberos TGT
08:06:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.031317 15418 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.031838 15418 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.032399 15418 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:31.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.058745 15418 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.059549 15418 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.14.110.3
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:31.089026 15418 tablet_server_runner.cc:78] Tablet server non-default flags:
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.3
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.3:0
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data/info.pb
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.3
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.14.110.3
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/logs
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:31.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:31.090116 15418 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:31.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:31.091012 15418 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.101028 15424 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:31.723183 15405 heartbeater.cc:499] Master 127.14.110.60:46255 was elected leader, sending a full tablet report...
08:06:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.102618 15425 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:31.105492 15427 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:32.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.295158 15426 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1191 milliseconds
08:06:32.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.295239 15418 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.299088 15418 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:32.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.301091 15418 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.302438 15418 hybrid_clock.cc:648] HybridClock initialized: now 1756800392302413 us; error 26 us; skew 500 ppm
08:06:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.303146 15418 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.308907 15418 init.cc:377] Logged in from keytab as kudu/127.14.110.3@KRBTEST.COM (short username kudu)
08:06:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.313606 15418 webserver.cc:480] Webserver started at http://127.14.110.3:44319/ using document root <none> and password file <none>
08:06:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.314545 15418 fs_manager.cc:362] Metadata directory not provided
08:06:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.314749 15418 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.315188 15418 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:06:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.318864 15418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data/instance:
08:06:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd58ff2ba754dc5b1c6f200d2c1374c"
08:06:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:32 on dist-test-slave-r4nz"
08:06:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.319927 15418 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal/instance:
08:06:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd58ff2ba754dc5b1c6f200d2c1374c"
08:06:32.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:32 on dist-test-slave-r4nz"
08:06:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.325997 15418 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
08:06:32.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.330438 15436 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.331292 15418 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
08:06:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.331588 15418 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal
08:06:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd58ff2ba754dc5b1c6f200d2c1374c"
08:06:32.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:32 on dist-test-slave-r4nz"
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.331862 15418 fs_report.cc:389] FS layout report
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data/data
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:32.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.374440 15418 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.380060 15418 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.380511 15418 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.383144 15418 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:06:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.386974 15418 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:06:32.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.387193 15418 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:06:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.387432 15418 ts_tablet_manager.cc:610] Registered 0 tablets
08:06:32.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.387586 15418 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:06:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.513584 15418 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.3:36363
08:06:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.513703 15548 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.3:36363 every 8 connection(s)
08:06:32.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.515841 15418 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/data/info.pb
08:06:32.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.520457 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15418
08:06:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.520884 14776 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/ts-2/wal/instance
08:06:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.3@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:32.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.3@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.573951 15550 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.574385 15550 heartbeater.cc:461] Registering TS with master...
08:06:32.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.575253 15550 heartbeater.cc:507] Master 127.14.110.60:46255 requested a full tablet report, sending...
08:06:32.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.3@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:32.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.577575 15006 ts_manager.cc:194] Registered new tserver with Master: 4bd58ff2ba754dc5b1c6f200d2c1374c (127.14.110.3:36363)
08:06:32.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.579202 15006 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.14.110.3@KRBTEST.COM'} at 127.14.110.3:39251
08:06:32.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.581455 15549 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:32.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.581718 15549 heartbeater.cc:461] Registering TS with master...
08:06:32.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.582324 15549 heartbeater.cc:507] Master 127.14.110.61:45735 requested a full tablet report, sending...
08:06:32.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.584656 14921 ts_manager.cc:194] Registered new tserver with Master: 4bd58ff2ba754dc5b1c6f200d2c1374c (127.14.110.3:36363)
08:06:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.590354 15551 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.590662 15551 heartbeater.cc:461] Registering TS with master...
08:06:32.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.591207 15551 heartbeater.cc:507] Master 127.14.110.62:34081 requested a full tablet report, sending...
08:06:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.593134 14840 ts_manager.cc:194] Registered new tserver with Master: 4bd58ff2ba754dc5b1c6f200d2c1374c (127.14.110.3:36363)
08:06:32.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.594647 14776 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:06:32.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.605 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:06:32.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:06:32.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:06:32.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.14.110.62"
port: 34081
}
bound_http_address {
host: "127.14.110.62"
port: 32877
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.14.110.61"
port: 45735
}
bound_http_address {
host: "127.14.110.61"
port: 40297
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.14.110.60"
port: 46255
}
bound_http_address {
host: "127.14.110.60"
port: 37537
}
}
}
08:06:32.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:06:32.617 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.14.110.1"
port: 41879
}
bound_http_address {
host: "127.14.110.1"
port: 45589
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.14.110.2"
port: 41493
}
bound_http_address {
host: "127.14.110.2"
port: 33523
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.14.110.3"
port: 36363
}
bound_http_address {
host: "127.14.110.3"
port: 44319
}
}
}
08:06:32.617 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:06:32.618 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:06:32.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:32.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:06:32.647 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.648 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc
08:06:32.653 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:06:32.654 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc
08:06:32.655 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:06:32.657 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:06:32.659 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:06:32.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:06:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:32 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800392, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:06:32.685 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.686 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc
08:06:32.687 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:06:32.687 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.110.61:45735
08:06:32.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:06:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.688755 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14884
08:06:32.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.710 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.110.62:34081
08:06:32.710 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:06:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.710795 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14805
08:06:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.711092 14990 connection.cc:537] client connection to 127.14.110.61:45735 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:06:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.711478 14990 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:06:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.714954 14990 consensus_peers.cc:489] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 -> Peer da8f4f913e4740e3a3b3770e4f291cbc (127.14.110.61:45735): Couldn't send request to peer da8f4f913e4740e3a3b3770e4f291cbc. Status: Network error: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:06:32.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.721045 15404 heartbeater.cc:646] Failed to heartbeat to 127.14.110.61:45735 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:32.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:32.736835 15292 connection.cc:537] client connection to 127.14.110.62:34081 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:06:32.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.737 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.110.60:46255
08:06:32.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:06:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.738186 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14966
08:06:32.766 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:32.766 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.110.61:45735
08:06:32.769 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:32.770479 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:32.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/logs
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.61:45735
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.61
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40297
08:06:32.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.61
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:32.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.052512 15567 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.052935 15567 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:33.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.053458 15567 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.079813 15567 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.080034 15567 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.080433 15567 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.080668 15567 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:33.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.080837 15567 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:33.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.080993 15567 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:33.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.081115 15567 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:33.111321 15567 master_runner.cc:386] Master server non-default flags:
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.61
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.61:45735
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.61
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40297
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/logs
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:33.112334 15567 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:33.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:33.113144 15567 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:33.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.123536 15573 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:33.582640 15550 heartbeater.cc:499] Master 127.14.110.60:46255 was elected leader, sending a full tablet report...
08:06:33.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.608958 15550 heartbeater.cc:646] Failed to heartbeat to 127.14.110.60:46255 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.678231 15257 heartbeater.cc:646] Failed to heartbeat to 127.14.110.61:45735 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.14.110.61:45735: connect: Connection refused (error 111)
08:06:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.964752 14200 debug-util.cc:398] Leaking SignalData structure 0x7b080009fd60 after lost signal to thread 14135
08:06:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.965646 14200 debug-util.cc:398] Leaking SignalData structure 0x7b080008f1e0 after lost signal to thread 14203
08:06:34.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.353542 14125 debug-util.cc:398] Leaking SignalData structure 0x7b08000a9700 after lost signal to thread 14062
08:06:34.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.354781 14125 debug-util.cc:398] Leaking SignalData structure 0x7b08000991a0 after lost signal to thread 14128
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.526405 15572 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 15567
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.762272 15572 kernel_stack_watchdog.cc:198] Thread 15567 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:33.124092 15574 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.764576 15575 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1639 milliseconds
08:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.764803 15576 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.763963 15567 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.640s user 0.551s sys 0.988s
08:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:34.765111 15567 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.641s user 0.552s sys 0.988s
08:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.765370 15567 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:34.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.768172 15567 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:34.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.770709 15567 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.772081 15567 hybrid_clock.cc:648] HybridClock initialized: now 1756800394771990 us; error 94 us; skew 500 ppm
08:06:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.772840 15567 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:34 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800394, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.778102 15567 init.cc:377] Logged in from keytab as kudu/127.14.110.61@KRBTEST.COM (short username kudu)
08:06:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.786521 15567 webserver.cc:480] Webserver started at http://127.14.110.61:40297/ using document root <none> and password file <none>
08:06:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.787446 15567 fs_manager.cc:362] Metadata directory not provided
08:06:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.787664 15567 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.795032 15567 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.003s
08:06:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.799360 15599 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:34.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.800328 15567 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
08:06:34.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.800635 15567 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:34.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da8f4f913e4740e3a3b3770e4f291cbc"
08:06:34.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:22 on dist-test-slave-r4nz"
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.802399 15567 fs_report.cc:389] FS layout report
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/wal
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/data
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:34.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.850962 15567 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.858980 15567 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:34.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.859402 15567 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:34.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.956105 15567 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.61:45735
08:06:34.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.956192 15650 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.61:45735 every 8 connection(s)
08:06:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.959590 15567 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-1/data/info.pb
08:06:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.962350 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15567
08:06:34.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:34.963 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.110.62:34081
08:06:34.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.965631 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/logs
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:34.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.62:34081
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.62
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32877
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.62
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.976200 15651 sys_catalog.cc:263] Verifying existing consensus state
08:06:34.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:34.984817 15651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Bootstrap starting.
08:06:35.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.040943 15651 log.cc:826] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Log is configured to *not* fsync() on all Append() calls
08:06:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.076663 15651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:06:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.077662 15651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Bootstrap complete.
08:06:35.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.111786 15651 raft_consensus.cc:357] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:35.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.129734 15651 raft_consensus.cc:738] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: da8f4f913e4740e3a3b3770e4f291cbc, State: Initialized, Role: FOLLOWER
08:06:35.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.132073 15651 consensus_queue.cc:260] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:35.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.142942 15657 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:35.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.143739 15657 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: FOLLOWER
08:06:35.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.144776 15651 sys_catalog.cc:564] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: configured and running, proceeding with master startup.
08:06:35.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.287501 15668 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:35.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.289055 15668 catalog_manager.cc:1554] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: loading cluster ID for follower catalog manager: success
08:06:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.304188 15668 catalog_manager.cc:1576] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: acquiring CA information for follower catalog manager: success
08:06:35.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.327327 15668 catalog_manager.cc:1604] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:35.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.272765 15567 master_runner.cc:186] Error getting master registration for 127.14.110.62:34081: Network error: Client connection negotiation failed: client connection to 127.14.110.62:34081: connect: Connection refused (error 111),
08:06:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.333216 15567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:35.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.513726 15654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.514449 15654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.515437 15654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.574775 15654 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.575073 15654 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.575523 15654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.575836 15654 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.576152 15654 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.576450 15654 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:35.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.576679 15654 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.645320 15654 master_runner.cc:386] Master server non-default flags:
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.62
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.62:34081
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.62
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32877
08:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/logs
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.647030 15654 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:35.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.648219 15654 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.689234 15680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:35.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.707124 15549 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.712788 15616 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4bd58ff2ba754dc5b1c6f200d2c1374c" instance_seqno: 1756800392484815) as {username='kudu', principal='kudu/127.14.110.3@KRBTEST.COM'} at 127.14.110.3:59635; Asking this server to re-register.
08:06:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.716218 15549 heartbeater.cc:461] Registering TS with master...
08:06:35.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.719457 15615 ts_manager.cc:194] Registered new tserver with Master: 4bd58ff2ba754dc5b1c6f200d2c1374c (127.14.110.3:36363)
08:06:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.753878 15257 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.756129 15615 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "7e301365419f41138fb4c260fdd8a1c5" instance_seqno: 1756800388360990) as {username='kudu', principal='kudu/127.14.110.1@KRBTEST.COM'} at 127.14.110.1:56477; Asking this server to re-register.
08:06:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.757524 15257 heartbeater.cc:461] Registering TS with master...
08:06:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.759989 15615 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:35.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.817452 15404 heartbeater.cc:344] Connected to a master server at 127.14.110.61:45735
08:06:35.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.820288 15615 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "beaa2681b4384ce89b2d29030fa685fc" instance_seqno: 1756800390628539) as {username='kudu', principal='kudu/127.14.110.2@KRBTEST.COM'} at 127.14.110.2:45893; Asking this server to re-register.
08:06:35.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.822831 15404 heartbeater.cc:461] Registering TS with master...
08:06:35.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:35.825780 15615 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:36.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:36.309716 15694 raft_consensus.cc:491] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:06:36.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:36.314669 15694 raft_consensus.cc:513] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:36.332671 15694 leader_election.cc:290] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081), 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255)
08:06:36.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:36.354979 15600 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.110.62:34081: connect: Connection refused (error 111)
08:06:36.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:36.376613 15600 leader_election.cc:336] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081): Network error: Client connection negotiation failed: client connection to 127.14.110.62:34081: connect: Connection refused (error 111)
08:06:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:36.382212 15600 leader_election.cc:336] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255): Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:36.384079 15600 leader_election.cc:304] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: da8f4f913e4740e3a3b3770e4f291cbc; no voters: 1abe593fbd60484eb94b87ce539701e2, 7635d667c7494f08a38d7a749f01d0e7
08:06:36.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:36.386530 15694 raft_consensus.cc:2747] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:35.690683 15678 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.067900 15676 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 15654
08:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.544431 15676 kernel_stack_watchdog.cc:198] Thread 15654 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 374ms:
08:06:37.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:37.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:37.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:37.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.548735 15654 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.856s user 0.001s sys 0.000s
08:06:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.549064 15654 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.857s user 0.002s sys 0.000s
08:06:37.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.549566 15681 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1857 milliseconds
08:06:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:37.550318 15683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:37.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.550388 15654 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.551517 15654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:37.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.554210 15654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.555613 15654 hybrid_clock.cc:648] HybridClock initialized: now 1756800397555531 us; error 88 us; skew 500 ppm
08:06:37.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.556288 15654 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:37 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800397, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.561522 15654 init.cc:377] Logged in from keytab as kudu/127.14.110.62@KRBTEST.COM (short username kudu)
08:06:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.567759 15654 webserver.cc:480] Webserver started at http://127.14.110.62:32877/ using document root <none> and password file <none>
08:06:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.568766 15654 fs_manager.cc:362] Metadata directory not provided
08:06:37.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.569018 15654 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:37.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.578639 15654 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
08:06:37.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.583954 15709 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.585109 15654 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:06:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.585427 15654 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:37.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:20 on dist-test-slave-r4nz"
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.587738 15654 fs_report.cc:389] FS layout report
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/wal
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/data
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:37.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.650566 15654 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:37.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.656314 15654 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:37.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.656735 15654 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.733315 15654 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.62:34081
08:06:37.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.733361 15761 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.62:34081 every 8 connection(s)
08:06:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.736678 15654 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-0/data/info.pb
08:06:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.745801 15762 sys_catalog.cc:263] Verifying existing consensus state
08:06:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.747032 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15654
08:06:37.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:37.748 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.110.60:46255
08:06:37.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.750568 14776 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/logs
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.60:46255
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.60
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37537
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.60
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:06:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.751797 15762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Bootstrap starting.
08:06:37.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.788254 15762 log.cc:826] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Log is configured to *not* fsync() on all Append() calls
08:06:37.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.812666 15762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:06:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.813562 15762 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: Bootstrap complete.
08:06:37.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.843420 15762 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.846199 15762 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1abe593fbd60484eb94b87ce539701e2, State: Initialized, Role: FOLLOWER
08:06:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.846825 15262 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.846956 15762 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.864656 15769 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.865578 15769 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.866287 15762 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: configured and running, proceeding with master startup.
08:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:37.886094 15406 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:38.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:38 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800398, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:38 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800397, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.014973 15140 init.cc:295] Successfully reacquired a new kerberos TGT
08:06:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.024752 15783 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.028426 15783 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: loading cluster ID for follower catalog manager: success
08:06:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.066790 15783 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: acquiring CA information for follower catalog manager: success
08:06:38.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:38 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800397, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.62@KRBTEST.COM for kudu/127.14.110.61@KRBTEST.COM
08:06:38.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.080709 15783 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:38.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.104863 15654 master_runner.cc:186] Error getting master registration for 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111),
08:06:38.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.127774 15654 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.184026 15764 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:06:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.184643 15764 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:06:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.185295 15764 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:06:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.212275 15764 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:06:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.212496 15764 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:06:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.212827 15764 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:06:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.213066 15764 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:06:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.213322 15764 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:06:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.213567 15764 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:06:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.213744 15764 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44201/jwks.json
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.245168 15764 master_runner.cc:386] Master server non-default flags:
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.110.20:39137
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.110.62:34081,127.14.110.61:45735,127.14.110.60:46255
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/krb5kdc/kudu.keytab
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.110.60
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.110.60:46255
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:06:38.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44201/jwks.json
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.110.60
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37537
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/logs
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Sep 2025 07:43:21 UTC on 5fd53c4cbb9d
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7861
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:06:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.246531 15764 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:06:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.247637 15764 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:06:38.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.277822 15793 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.576388 15799 raft_consensus.cc:491] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:06:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.576777 15799 raft_consensus.cc:513] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.605552 15799 leader_election.cc:290] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081), 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255)
08:06:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.635097 15600 leader_election.cc:336] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255): Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:38.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.636332 15737 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1abe593fbd60484eb94b87ce539701e2" is_pre_election: true
08:06:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.637039 15737 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate da8f4f913e4740e3a3b3770e4f291cbc in term 1.
08:06:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.638176 15600 leader_election.cc:304] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 1abe593fbd60484eb94b87ce539701e2, da8f4f913e4740e3a3b3770e4f291cbc; no voters: 7635d667c7494f08a38d7a749f01d0e7
08:06:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.638768 15799 raft_consensus.cc:2802] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Leader pre-election won for term 2
08:06:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.639032 15799 raft_consensus.cc:491] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:06:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.639308 15799 raft_consensus.cc:3058] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 1 FOLLOWER]: Advancing to term 2
08:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.647699 15799 raft_consensus.cc:513] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.663362 15737 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1abe593fbd60484eb94b87ce539701e2"
08:06:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.663800 15737 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 1 FOLLOWER]: Advancing to term 2
08:06:38.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.671439 15737 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate da8f4f913e4740e3a3b3770e4f291cbc in term 2.
08:06:38.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.672313 15600 leader_election.cc:304] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1abe593fbd60484eb94b87ce539701e2, da8f4f913e4740e3a3b3770e4f291cbc; no voters:
08:06:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.673877 15799 leader_election.cc:290] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 election: Requested vote from peers 1abe593fbd60484eb94b87ce539701e2 (127.14.110.62:34081), 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255)
08:06:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.678249 15799 raft_consensus.cc:2802] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 2 FOLLOWER]: Leader election won for term 2
08:06:38.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.678723 15799 raft_consensus.cc:695] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [term 2 LEADER]: Becoming Leader. State: Replica: da8f4f913e4740e3a3b3770e4f291cbc, State: Running, Role: LEADER
08:06:38.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.679764 15799 consensus_queue.cc:237] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.687906 15600 leader_election.cc:336] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255): Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111)
08:06:38.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.699555 15804 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: New leader da8f4f913e4740e3a3b3770e4f291cbc. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:38.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.700163 15804 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: LEADER
08:06:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.709797 15807 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:06:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.724692 15551 heartbeater.cc:344] Connected to a master server at 127.14.110.62:34081
08:06:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.730432 15726 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "4bd58ff2ba754dc5b1c6f200d2c1374c" instance_seqno: 1756800392484815) as {username='kudu', principal='kudu/127.14.110.3@KRBTEST.COM'} at 127.14.110.3:34359; Asking this server to re-register.
08:06:38.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.733665 15551 heartbeater.cc:461] Registering TS with master...
08:06:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.736792 15726 ts_manager.cc:194] Registered new tserver with Master: 4bd58ff2ba754dc5b1c6f200d2c1374c (127.14.110.3:36363)
08:06:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.740258 15807 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:06:38.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.758882 15807 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:38.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.763378 15807 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:06:38.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.782109 15807 catalog_manager.cc:1506] Loading token signing keys...
08:06:38.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.799853 15807 catalog_manager.cc:5966] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc: Loaded TSK: 0
08:06:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.803807 15807 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:06:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.857920 15726 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "7e301365419f41138fb4c260fdd8a1c5" instance_seqno: 1756800388360990) as {username='kudu', principal='kudu/127.14.110.1@KRBTEST.COM'} at 127.14.110.1:49119; Asking this server to re-register.
08:06:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.859274 15262 heartbeater.cc:461] Registering TS with master...
08:06:38.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.862309 15726 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.891325 15726 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "beaa2681b4384ce89b2d29030fa685fc" instance_seqno: 1756800390628539) as {username='kudu', principal='kudu/127.14.110.2@KRBTEST.COM'} at 127.14.110.2:59235; Asking this server to re-register.
08:06:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.892613 15406 heartbeater.cc:461] Registering TS with master...
08:06:38.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:38.895298 15726 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.966599 14478 debug-util.cc:398] Leaking SignalData structure 0x7b08000acd80 after lost signal to thread 14348
08:06:38.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.967585 14478 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3b60 after lost signal to thread 14481
08:06:39.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:39.174062 15600 consensus_peers.cc:489] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc -> Peer 7635d667c7494f08a38d7a749f01d0e7 (127.14.110.60:46255): Couldn't send request to peer 7635d667c7494f08a38d7a749f01d0e7. Status: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:06:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.267505 15737 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [term 2 FOLLOWER]: Refusing update from remote peer da8f4f913e4740e3a3b3770e4f291cbc: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:06:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.269318 15804 consensus_queue.cc:1035] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [LEADER]: Connected to new peer: Peer: permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:06:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.314654 15812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader da8f4f913e4740e3a3b3770e4f291cbc. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.315284 15812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:39.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.336751 15812 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.337473 15812 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1abe593fbd60484eb94b87ce539701e2 [sys.catalog]: This master's current role is: FOLLOWER
08:06:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.348569 15804 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.349262 15804 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: LEADER
08:06:39.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.837684 15257 heartbeater.cc:499] Master 127.14.110.61:45735 was elected leader, sending a full tablet report...
08:06:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:39.887112 15404 heartbeater.cc:499] Master 127.14.110.61:45735 was elected leader, sending a full tablet report...
08:06:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:40.383281 15567 master_runner.cc:186] Error getting master registration for 127.14.110.60:46255: Network error: Client connection negotiation failed: client connection to 127.14.110.60:46255: connect: Connection refused (error 111),
08:06:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:40.383776 15567 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:06:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:40 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800400, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:40.444270 15287 init.cc:295] Successfully reacquired a new kerberos TGT
08:06:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:40.743484 15549 heartbeater.cc:499] Master 127.14.110.61:45735 was elected leader, sending a full tablet report...
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:39.673034 15792 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 15764
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:41.326153 15792 kernel_stack_watchdog.cc:198] Thread 15764 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 384ms:
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:06:41.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:06:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:38.287127 15794 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:41.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:41.328856 15797 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:06:41.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:41.329367 15764 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.042s user 0.608s sys 1.318s
08:06:41.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:41.329795 15764 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.043s user 0.609s sys 1.318s
08:06:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:41.331140 15795 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3041 milliseconds
08:06:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.331195 15764 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:06:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.332381 15764 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:06:41.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.335013 15764 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:06:41.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.336376 15764 hybrid_clock.cc:648] HybridClock initialized: now 1756800401336289 us; error 83 us; skew 500 ppm
08:06:41.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.337051 15764 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:06:41.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:41 dist-test-slave-r4nz krb5kdc[14788](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800401, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:06:41.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.342759 15764 init.cc:377] Logged in from keytab as kudu/127.14.110.60@KRBTEST.COM (short username kudu)
08:06:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.348302 15764 webserver.cc:480] Webserver started at http://127.14.110.60:37537/ using document root <none> and password file <none>
08:06:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.349093 15764 fs_manager.cc:362] Metadata directory not provided
08:06:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.349318 15764 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:06:41.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.356384 15764 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.003s sys 0.003s
08:06:41.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.360477 15837 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:06:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.362419 15764 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
08:06:41.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.362867 15764 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data,/tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:41.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7635d667c7494f08a38d7a749f01d0e7"
08:06:41.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-02 08:06:24 on dist-test-slave-r4nz"
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.366027 15764 fs_report.cc:389] FS layout report
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/wal
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/data
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:06:41.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:06:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:06:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:06:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:06:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.451568 15764 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:06:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.482381 15764 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:06:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.483101 15764 kserver.cc:163] Server-wide thread pool size limit: 3276
08:06:41.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.561682 15764 rpc_server.cc:307] RPC server started. Bound to: 127.14.110.60:46255
08:06:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.561750 15888 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.110.60:46255 every 8 connection(s)
08:06:41.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.564204 15764 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskA19hJ6/test-tmp/mini-kudu-cluster7412850666087634840/master-2/data/info.pb
08:06:41.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.566695 14776 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu as pid 15764
08:06:41.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:06:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.578387 15889 sys_catalog.cc:263] Verifying existing consensus state
08:06:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.584960 15889 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Bootstrap starting.
08:06:41.615 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.110.61:45735(127.14.110.61:45735)] connecting to peer
08:06:41.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.640731 15889 log.cc:826] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Log is configured to *not* fsync() on all Append() calls
08:06:41.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:41 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800394, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.61@KRBTEST.COM for kudu/127.14.110.60@KRBTEST.COM
08:06:41.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.682253 15889 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:06:41.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.683089 15889 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: Bootstrap complete.
08:06:41.695 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.110.62:34081(127.14.110.62:34081)] connecting to peer
08:06:41.701 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.110.60:46255(127.14.110.60:46255)] connecting to peer
08:06:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.707651 15889 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.710343 15889 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7635d667c7494f08a38d7a749f01d0e7, State: Initialized, Role: FOLLOWER
08:06:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.710945 15889 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } }
08:06:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.713230 15896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.713980 15896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: FOLLOWER
08:06:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.714649 15889 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: configured and running, proceeding with master startup.
08:06:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.715366 15864 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [term 1 FOLLOWER]: Advancing to term 2
08:06:41.728 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.110.61:45735(127.14.110.61:45735)] Successfully connected to peer
08:06:41.731 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.14.110.62:34081(127.14.110.62:34081)] Successfully connected to peer
08:06:41.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.786964 15550 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:41.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.787376 15550 heartbeater.cc:499] Master 127.14.110.60:46255 was elected leader, sending a full tablet report...
08:06:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.790699 15896 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader da8f4f913e4740e3a3b3770e4f291cbc. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.791533 15896 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: FOLLOWER
08:06:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.792528 15910 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.793370 15910 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7 [sys.catalog]: This master's current role is: FOLLOWER
08:06:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.810423 15914 catalog_manager.cc:1261] Loaded cluster ID: 404c5529eea946baaf078a610ad470b4
08:06:41.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.810708 15914 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: loading cluster ID for follower catalog manager: success
08:06:41.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 02 08:06:41 dist-test-slave-r4nz krb5kdc[14788](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756800401, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.110.60@KRBTEST.COM for kudu/127.14.110.62@KRBTEST.COM
08:06:41.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.817004 15914 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: acquiring CA information for follower catalog manager: success
08:06:41.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.821846 15914 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 7635d667c7494f08a38d7a749f01d0e7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:06:41.832 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.110.60:46255(127.14.110.60:46255)] Successfully connected to peer
08:06:41.858 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:06:41.869 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.110.62
08:06:41.869 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.110.61
08:06:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.912437 15258 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.919314 15854 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "7e301365419f41138fb4c260fdd8a1c5" instance_seqno: 1756800388360990) as {username='kudu', principal='kudu/127.14.110.1@KRBTEST.COM'} at 127.14.110.1:55017; Asking this server to re-register.
08:06:41.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.923974 15258 heartbeater.cc:461] Registering TS with master...
08:06:41.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.927129 15853 ts_manager.cc:194] Registered new tserver with Master: 7e301365419f41138fb4c260fdd8a1c5 (127.14.110.1:41879)
08:06:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.964843 15405 heartbeater.cc:344] Connected to a master server at 127.14.110.60:46255
08:06:41.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.966976 15853 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "beaa2681b4384ce89b2d29030fa685fc" instance_seqno: 1756800390628539) as {username='kudu', principal='kudu/127.14.110.2@KRBTEST.COM'} at 127.14.110.2:41271; Asking this server to re-register.
08:06:41.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.968142 15405 heartbeater.cc:461] Registering TS with master...
08:06:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:41.970785 15853 ts_manager.cc:194] Registered new tserver with Master: beaa2681b4384ce89b2d29030fa685fc (127.14.110.2:41493)
08:06:42.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.110.60:46255(127.14.110.60:46255)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
08:06:42.022 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.110.60:46255: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
08:06:42.023 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.110.60:46255(127.14.110.60:46255)] handling channelInactive
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:42.024966 15907 negotiation.cc:337] Failed RPC negotiation. Trace:
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.729568 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53280 (local address 127.14.110.60:46255)
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.782234 (+ 52666us) server_negotiation.cc:206] Beginning negotiation
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.782256 (+ 22us) server_negotiation.cc:399] Waiting for connection header
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.833226 (+ 50970us) server_negotiation.cc:407] Connection header received
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.834050 (+ 824us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.834074 (+ 24us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.834289 (+ 215us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.834469 (+ 180us) server_negotiation.cc:226] Negotiated authn=SASL
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.023225 (+188756us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53280: BlockingRecv error: recv got EOF from 127.0.0.1:53280 (error 108)
08:06:42.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":51858,"thread_start_us":52515,"threads_started":1}
08:06:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.111631 15804 sys_catalog.cc:455] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1abe593fbd60484eb94b87ce539701e2" member_type: VOTER last_known_addr { host: "127.14.110.62" port: 34081 } } peers { permanent_uuid: "da8f4f913e4740e3a3b3770e4f291cbc" member_type: VOTER last_known_addr { host: "127.14.110.61" port: 45735 } } peers { permanent_uuid: "7635d667c7494f08a38d7a749f01d0e7" member_type: VOTER last_known_addr { host: "127.14.110.60" port: 46255 } } }
08:06:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.112493 15804 sys_catalog.cc:458] T 00000000000000000000000000000000 P da8f4f913e4740e3a3b3770e4f291cbc [sys.catalog]: This master's current role is: LEADER
08:06:42.122 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.110.62:34081(127.14.110.62:34081)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:06:42.122 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.110.61:45735(127.14.110.61:45735)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:06:42.123 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.110.62:34081: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:06:42.127 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.110.61:45735: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:42.130520 15902 negotiation.cc:337] Failed RPC negotiation. Trace:
08:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.724611 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39004 (local address 127.14.110.62:34081)
08:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.738391 (+ 13780us) server_negotiation.cc:206] Beginning negotiation
08:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.738425 (+ 34us) server_negotiation.cc:399] Waiting for connection header
08:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.805788 (+ 67363us) server_negotiation.cc:407] Connection header received
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.831781 (+ 25993us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.831807 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.831994 (+ 187us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.832148 (+ 154us) server_negotiation.cc:226] Negotiated authn=SASL
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.008664 (+176516us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.010354 (+ 1690us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.113506 (+103152us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.114189 (+ 683us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.129027 (+ 14838us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39004: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:39004): Cannot send after transport endpoint shutdown (error 108)
08:06:42.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":13271,"thread_start_us":13760,"threads_started":1}
08:06:42.131 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.14.110.62:34081(127.14.110.62:34081)] handling channelInactive
08:06:42.134 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=563), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.61:45735, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.62:34081, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.60:46255, [444ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.60:46255: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.62:34081: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.61:45735: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.14.110.61:45735,127.14.110.62:34081,127.14.110.60:46255). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.14.110.61:45735,127.14.110.62:34081,127.14.110.60:46255). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
08:06:42.139 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=568), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.61:45735, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.62:34081, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.110.60:46255, [444ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.60:46255: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.62:34081: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.110.61:45735: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
08:06:42.140 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.110.61:45735(127.14.110.61:45735)] handling channelInactive
08:06:42.141 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.110.62:34081(127.14.110.62:34081)] disconnecting while in state TERMINATED
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:42.141785 15903 negotiation.cc:337] Failed RPC negotiation. Trace:
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.725144 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42742 (local address 127.14.110.61:45735)
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.726311 (+ 1167us) server_negotiation.cc:206] Beginning negotiation
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.726338 (+ 27us) server_negotiation.cc:399] Waiting for connection header
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.805744 (+ 79406us) server_negotiation.cc:407] Connection header received
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.831814 (+ 26070us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.831845 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.832048 (+ 203us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:41.832206 (+ 158us) server_negotiation.cc:226] Negotiated authn=SASL
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.016891 (+184685us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.018570 (+ 1679us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.110359 (+ 91789us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.110880 (+ 521us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:06:42.142 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.110.60:46255(127.14.110.60:46255)] disconnecting while in state TERMINATED
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0902 08:06:42.140362 (+ 29482us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42742: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:42742): Cannot send after transport endpoint shutdown (error 108)
08:06:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":644,"thread_start_us":454,"threads_started":1}
08:06:42.142 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.14.110.61:45735(127.14.110.61:45735)] disconnecting while in state TERMINATED
08:06:42.147 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:06:42.149 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:06:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.155328 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15097
08:06:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.186609 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15271
08:06:42.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.211022 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15418
08:06:42.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.239746 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15654
08:06:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.282724 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15567
08:06:42.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:42.312048 14776 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 15764
08:06:43.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:43Z chronyd exiting
08:06:44.453 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:06:44.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.454360 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14329
08:06:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.476240 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14492
08:06:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.499666 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14633
08:06:44.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.522816 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14061
08:06:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.547535 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14131
08:06:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:44.573220 14234 connection.cc:537] client connection to 127.13.184.125:41845 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:06:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250902 08:06:44.573653 14049 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskA19hJ6/build/tsan/bin/kudu with pid 14206
08:06:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250902 08:06:44.573679 14234 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:06:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-02T08:06:45Z chronyd exiting
08:06:46.593 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.14.110.61:45735,127.14.110.62:34081,127.14.110.60:46255). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.14.110.61:45735,127.14.110.62:34081,127.14.110.60:46255). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more