02:07:47.616 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:07:47.628 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskX8607x/test-tmp
02:07:47.632 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:07:47.635 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskX8607x/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:07:47.877 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:07:48.034 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:07:48.036 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:07:48.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.039642 19545 mini_oidc.cc:150] Starting JWKS server
02:07:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.043510 19545 webserver.cc:480] Webserver started at http://127.0.0.1:36237/ using document root <none> and password file <none>
02:07:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.043972 19545 mini_oidc.cc:186] Starting OIDC Discovery server
02:07:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.045912 19545 webserver.cc:480] Webserver started at http://127.0.0.1:43195/ using document root <none> and password file <none>
02:07:48.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:07:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:07:48.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:07:48Z Disabled control of system clock
02:07:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.108213 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:48.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/logs
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data/info.pb
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.126:46717
02:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.126
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:48.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json with env {}
02:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.414865 19557 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.415453 19557 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.416019 19557 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:48.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.448243 19557 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:07:48.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.448544 19557 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.448741 19557 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.448997 19557 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:07:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.449270 19557 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:48.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.449415 19557 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36237/jwks.json
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.485569 19557 master_runner.cc:386] Master server non-default flags:
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:48.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data
02:07:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal
02:07:48.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.126:46717
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json
02:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data/info.pb
02:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.126
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:48.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/logs
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:07:48.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:07:48.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:48.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.487066 19557 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:48.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:48.488938 19557 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:48.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.500881 19563 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:49.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:48.502074 19564 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:49.824677 19565 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1320 milliseconds
02:07:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:49.826225 19566 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:49.828980 19557 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.328s user 0.398s sys 0.922s
02:07:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:49.829454 19557 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.328s user 0.402s sys 0.922s
02:07:49.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.829788 19557 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:07:49.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.835363 19557 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:49.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.839071 19557 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:49.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.841022 19557 hybrid_clock.cc:648] HybridClock initialized: now 1756606069840937 us; error 61 us; skew 500 ppm
02:07:49.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.842258 19557 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:49.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.852217 19557 webserver.cc:480] Webserver started at http://127.19.22.126:32985/ using document root <none> and password file <none>
02:07:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.853659 19557 fs_manager.cc:362] Metadata directory not provided
02:07:49.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.853986 19557 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.854626 19557 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:07:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.862600 19557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data/instance:
02:07:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17196fcbdd6c4beca1694c6a41db19b9"
02:07:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:49 on dist-test-slave-kjk0"
02:07:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.864614 19557 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal/instance:
02:07:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17196fcbdd6c4beca1694c6a41db19b9"
02:07:49.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:49 on dist-test-slave-kjk0"
02:07:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.875825 19557 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
02:07:49.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.884541 19573 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:49.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.886111 19557 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
02:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.886603 19557 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal
02:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17196fcbdd6c4beca1694c6a41db19b9"
02:07:49.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:49 on dist-test-slave-kjk0"
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.887113 19557 fs_report.cc:389] FS layout report
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data/data
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:49.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:49.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:49.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.964275 19557 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.965862 19557 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:49.966303 19557 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.056332 19557 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.126:46717
02:07:50.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.056334 19624 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.126:46717 every 8 connection(s)
02:07:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.059948 19557 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/data/info.pb
02:07:50.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.065850 19625 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:50.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.069782 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 19557
02:07:50.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.070183 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-0/wal/instance
02:07:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.074254 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/logs
02:07:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data/info.pb
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.125:33753
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.125
02:07:50.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json with env {}
02:07:50.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.081314 19625 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
02:07:50.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.111526 19625 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } has no permanent_uuid. Determining permanent_uuid...
02:07:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.114076 19575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.118726 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.168354 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 1
02:07:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.173122 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.241525 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 2
02:07:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.245925 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.337386 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 3
02:07:50.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.341348 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.396018 19627 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:50.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.396579 19627 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:50.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.397094 19627 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:50.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.428459 19627 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:07:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.428771 19627 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.428977 19627 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.429191 19627 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:07:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.429371 19627 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.429518 19627 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36237/jwks.json
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.465938 19627 master_runner.cc:386] Master server non-default flags:
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal
02:07:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.125:33753
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data/info.pb
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.125
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:50.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/logs
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.467376 19627 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.469813 19627 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.482782 19636 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:50.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.485132 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 4
02:07:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.492961 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:50.792640 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 5
02:07:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.800786 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.348376 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 6
02:07:51.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.375927 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.125:33753: Network error: Client connection negotiation failed: client connection to 127.19.22.125:33753: connect: Connection refused (error 111)
02:07:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.886008 19635 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 19627
02:07:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.923000 19635 kernel_stack_watchdog.cc:198] Thread 19627 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.923529 19627 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.441s user 0.487s sys 0.942s
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.923939 19627 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.442s user 0.487s sys 0.942s
02:07:51.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.924297 19638 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1440 milliseconds
02:07:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:50.482900 19637 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.926131 19627 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:07:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:51.926147 19639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:51.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.929687 19627 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:51.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.932319 19627 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.933686 19627 hybrid_clock.cc:648] HybridClock initialized: now 1756606071933636 us; error 57 us; skew 500 ppm
02:07:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.934616 19627 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.941254 19627 webserver.cc:480] Webserver started at http://127.19.22.125:34583/ using document root <none> and password file <none>
02:07:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.942307 19627 fs_manager.cc:362] Metadata directory not provided
02:07:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.942545 19627 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:51.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.943101 19627 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:07:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.947837 19627 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data/instance:
02:07:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7930b8a7cfc94aa6872e038bae3caf2d"
02:07:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:51 on dist-test-slave-kjk0"
02:07:51.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.949374 19627 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal/instance:
02:07:51.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7930b8a7cfc94aa6872e038bae3caf2d"
02:07:51.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:51 on dist-test-slave-kjk0"
02:07:51.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.957384 19627 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
02:07:51.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.963351 19648 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:51.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.964512 19627 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:07:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.964900 19627 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal
02:07:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7930b8a7cfc94aa6872e038bae3caf2d"
02:07:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:51 on dist-test-slave-kjk0"
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:51.965340 19627 fs_report.cc:389] FS layout report
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data/data
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.016970 19627 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.018638 19627 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.019183 19627 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.095552 19627 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.125:33753
02:07:52.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.095618 19699 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.125:33753 every 8 connection(s)
02:07:52.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.098505 19627 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/data/info.pb
02:07:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.102707 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 19627
02:07:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.103226 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-1/wal/instance
02:07:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.104737 19700 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.106292 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal
02:07:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data
02:07:52.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/logs
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data/info.pb
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:07:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.124:38429
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.124
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json with env {}
02:07:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.118496 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
02:07:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.140744 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } has no permanent_uuid. Determining permanent_uuid...
02:07:52.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.156744 19700 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
02:07:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.159422 19649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.164526 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.214136 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 1
02:07:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.218950 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.287456 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 2
02:07:52.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.291882 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.383386 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 3
02:07:52.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.387619 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.426337 19702 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:52.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.427047 19702 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.427736 19702 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.436578 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } attempt: 7
02:07:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.448238 19625 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
02:07:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.454289 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.464323 19702 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:07:52.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.464620 19702 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.464813 19702 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:07:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.464996 19702 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:07:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.465276 19702 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:07:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.465430 19702 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36237/jwks.json
02:07:52.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.501438 19702 master_runner.cc:386] Master server non-default flags:
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:52.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.124:38429
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36237/jwks.json
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data/info.pb
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.124
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:52.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/logs
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:07:52.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:52.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.502904 19702 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:52.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.505306 19702 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.512815 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 1
02:07:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.518599 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.520613 19714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.531003 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 4
02:07:52.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.556089 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.600068 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 2
02:07:52.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.604274 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.689699 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 3
02:07:52.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.696769 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.837491 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 4
02:07:52.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.844583 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:52.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:52.861687 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 5
02:07:52.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.868654 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:53.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.128229 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 5
02:07:53.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.135449 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.416303 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 6
02:07:53.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.429189 19700 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.570037 19696 debug-util.cc:398] Leaking SignalData structure 0x7b080006f060 after lost signal to thread 19631
02:07:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.570744 19696 debug-util.cc:398] Leaking SignalData structure 0x7b080006f7a0 after lost signal to thread 19699
02:07:53.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.688144 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 6
02:07:53.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.712041 19625 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.22.124:38429: Network error: Client connection negotiation failed: client connection to 127.19.22.124:38429: connect: Connection refused (error 111)
02:07:53.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.923945 19713 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 19702
02:07:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.950743 19702 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.432s user 0.000s sys 0.002s
02:07:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.952270 19702 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.433s user 0.001s sys 0.002s
02:07:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:52.521342 19715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:53.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.959323 19702 server_base.cc:1047] running on GCE node
02:07:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:53.959916 19723 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:53.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.961205 19702 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.964262 19702 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:53.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.965945 19702 hybrid_clock.cc:648] HybridClock initialized: now 1756606073965893 us; error 53 us; skew 500 ppm
02:07:53.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.966851 19702 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.974618 19702 webserver.cc:480] Webserver started at http://127.19.22.124:36929/ using document root <none> and password file <none>
02:07:53.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.975632 19702 fs_manager.cc:362] Metadata directory not provided
02:07:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.975857 19702 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.976415 19702 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:07:53.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.981140 19702 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data/instance:
02:07:53.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6619257b90774cf3acfa543fbf92f6cd"
02:07:53.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:53 on dist-test-slave-kjk0"
02:07:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.982406 19702 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal/instance:
02:07:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6619257b90774cf3acfa543fbf92f6cd"
02:07:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:53 on dist-test-slave-kjk0"
02:07:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.990940 19702 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.001s sys 0.008s
02:07:53.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.997382 19728 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.998554 19702 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:07:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.998916 19702 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal
02:07:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6619257b90774cf3acfa543fbf92f6cd"
02:07:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:53 on dist-test-slave-kjk0"
02:07:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:53.999321 19702 fs_report.cc:389] FS layout report
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data/data
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.086827 19702 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.088976 19702 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.090068 19702 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.180938 19779 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.124:38429 every 8 connection(s)
02:07:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.180939 19702 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.124:38429
02:07:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.185775 19702 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/data/info.pb
02:07:54.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.192111 19780 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:07:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.194310 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 19702
02:07:54.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.194815 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/master-2/wal/instance
02:07:54.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.211695 19780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } has no permanent_uuid. Determining permanent_uuid...
02:07:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.253504 19780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } has no permanent_uuid. Determining permanent_uuid...
02:07:54.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.268791 19780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
02:07:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.291778 19780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: Bootstrap starting.
02:07:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.300670 19780 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: Neither blocks nor log segments found. Creating new log.
02:07:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.303460 19780 log.cc:826] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: Log is configured to *not* fsync() on all Append() calls
02:07:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.309780 19780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: No bootstrap required, opened a new log
02:07:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.340030 19780 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.340678 19780 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.340871 19780 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6619257b90774cf3acfa543fbf92f6cd, State: Initialized, Role: FOLLOWER
02:07:54.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.341642 19780 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.344269 19789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.345470 19780 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: configured and running, proceeding with master startup.
02:07:54.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.345474 19789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: This master's current role is: FOLLOWER
02:07:54.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.347105 19734 tablet.cc:2378] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:54.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.381946 19800 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:54.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.382784 19800 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.374258 19702 master_runner.cc:186] Error getting master registration for 127.19.22.126:46717: OK, instance_id { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" instance_seqno: 1756606070029936 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:07:54.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.409044 19702 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:54.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.489781 19700 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 7
02:07:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.517302 19700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: Bootstrap starting.
02:07:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.526289 19700 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: Neither blocks nor log segments found. Creating new log.
02:07:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.528964 19700 log.cc:826] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: Log is configured to *not* fsync() on all Append() calls
02:07:54.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.534858 19700 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: No bootstrap required, opened a new log
02:07:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.565956 19700 raft_consensus.cc:357] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.566951 19700 raft_consensus.cc:383] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.567281 19700 raft_consensus.cc:738] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7930b8a7cfc94aa6872e038bae3caf2d, State: Initialized, Role: FOLLOWER
02:07:54.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.568301 19700 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.576562 19804 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.577946 19804 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: This master's current role is: FOLLOWER
02:07:54.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.580287 19700 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: configured and running, proceeding with master startup.
02:07:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.635946 19815 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:07:54.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.642379 19815 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:07:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.653281 19804 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:07:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.653870 19804 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.659595 19804 leader_election.cc:290] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 17196fcbdd6c4beca1694c6a41db19b9 (127.19.22.126:46717), 6619257b90774cf3acfa543fbf92f6cd (127.19.22.124:38429)
02:07:54.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.621510 19627 master_runner.cc:186] Error getting master registration for 127.19.22.126:46717: OK, instance_id { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" instance_seqno: 1756606070029936 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:07:54.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.661523 19627 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:07:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.661446 19755 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6619257b90774cf3acfa543fbf92f6cd" is_pre_election: true
02:07:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.662406 19755 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7930b8a7cfc94aa6872e038bae3caf2d in term 0.
02:07:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.665164 19649 leader_election.cc:304] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6619257b90774cf3acfa543fbf92f6cd, 7930b8a7cfc94aa6872e038bae3caf2d; no voters:
02:07:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.666123 19804 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Leader pre-election won for term 1
02:07:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.666483 19804 raft_consensus.cc:491] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:07:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.666837 19804 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 0 FOLLOWER]: Advancing to term 1
02:07:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.674695 19804 raft_consensus.cc:513] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.678030 19804 leader_election.cc:290] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [CANDIDATE]: Term 1 election: Requested vote from peers 17196fcbdd6c4beca1694c6a41db19b9 (127.19.22.126:46717), 6619257b90774cf3acfa543fbf92f6cd (127.19.22.124:38429)
02:07:54.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.678653 19755 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6619257b90774cf3acfa543fbf92f6cd"
02:07:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.679729 19755 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 0 FOLLOWER]: Advancing to term 1
02:07:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.687603 19755 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7930b8a7cfc94aa6872e038bae3caf2d in term 1.
02:07:54.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.688915 19649 leader_election.cc:304] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6619257b90774cf3acfa543fbf92f6cd, 7930b8a7cfc94aa6872e038bae3caf2d; no voters:
02:07:54.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.691955 19804 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 1 FOLLOWER]: Leader election won for term 1
02:07:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.705054 19804 raft_consensus.cc:695] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [term 1 LEADER]: Becoming Leader. State: Replica: 7930b8a7cfc94aa6872e038bae3caf2d, State: Running, Role: LEADER
02:07:54.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.706688 19804 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.724637 19817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7930b8a7cfc94aa6872e038bae3caf2d. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.725973 19817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: This master's current role is: LEADER
02:07:54.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.731967 19819 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:07:54.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.741446 19819 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:07:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.745556 19625 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } attempt: 7
02:07:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.777424 19654 tablet.cc:2378] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.802126 19755 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [term 1 FOLLOWER]: Refusing update from remote peer 7930b8a7cfc94aa6872e038bae3caf2d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:07:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.804423 19804 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:07:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.818372 19625 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: Bootstrap starting.
02:07:54.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.843984 19625 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: Neither blocks nor log segments found. Creating new log.
02:07:54.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.856772 19789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7930b8a7cfc94aa6872e038bae3caf2d. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.862138 19789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: This master's current role is: FOLLOWER
02:07:54.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.863556 19625 log.cc:826] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: Log is configured to *not* fsync() on all Append() calls
02:07:54.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.879503 19822 mvcc.cc:204] Tried to move back new op lower bound from 7195058482342846464 to 7195058482011586560. Current Snapshot: MvccSnapshot[applied={T|T < 7195058482342846464}]
02:07:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.901536 19789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.902333 19789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd [sys.catalog]: This master's current role is: FOLLOWER
02:07:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.911700 19817 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.912468 19817 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: This master's current role is: LEADER
02:07:54.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.921377 19625 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: No bootstrap required, opened a new log
02:07:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.933270 19819 catalog_manager.cc:1349] Generated new cluster ID: 2faed0373fde44a7a181f5a262b47b0d
02:07:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.933620 19819 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:07:54.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.951102 19625 raft_consensus.cc:357] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.952332 19625 raft_consensus.cc:383] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:07:54.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.953552 19625 raft_consensus.cc:738] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 17196fcbdd6c4beca1694c6a41db19b9, State: Initialized, Role: FOLLOWER
02:07:54.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.954833 19625 consensus_queue.cc:260] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } }
02:07:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.968024 19827 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:54.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.969708 19827 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: This master's current role is: FOLLOWER
02:07:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.970832 19625 sys_catalog.cc:564] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: configured and running, proceeding with master startup.
02:07:54.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.972836 19598 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [term 0 FOLLOWER]: Advancing to term 1
02:07:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.971007 19600 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "17196fcbdd6c4beca1694c6a41db19b9" is_pre_election: true
02:07:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.971742 19599 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "17196fcbdd6c4beca1694c6a41db19b9"
02:07:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:54.977530 19579 tablet.cc:2378] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:07:54.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.984388 19598 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [term 1 FOLLOWER]: Refusing update from remote peer 7930b8a7cfc94aa6872e038bae3caf2d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:07:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.986388 19817 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [LEADER]: Connected to new peer: Peer: permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.991395 19819 catalog_manager.cc:1372] Generated new certificate authority record
02:07:54.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:54.993546 19819 catalog_manager.cc:1506] Loading token signing keys...
02:07:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.031445 19829 mvcc.cc:204] Tried to move back new op lower bound from 7195058482342846464 to 7195058482011586560. Current Snapshot: MvccSnapshot[applied={T|T < 7195058482342846464}]
02:07:55.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.050351 19827 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7930b8a7cfc94aa6872e038bae3caf2d. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:55.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.051492 19827 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.055660 19819 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d: Generated new TSK 0
02:07:55.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.061751 19831 sys_catalog.cc:455] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:55.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.066592 19819 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:07:55.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.070135 19804 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "17196fcbdd6c4beca1694c6a41db19b9" member_type: VOTER last_known_addr { host: "127.19.22.126" port: 46717 } } peers { permanent_uuid: "7930b8a7cfc94aa6872e038bae3caf2d" member_type: VOTER last_known_addr { host: "127.19.22.125" port: 33753 } } peers { permanent_uuid: "6619257b90774cf3acfa543fbf92f6cd" member_type: VOTER last_known_addr { host: "127.19.22.124" port: 38429 } } }
02:07:55.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.070861 19804 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7930b8a7cfc94aa6872e038bae3caf2d [sys.catalog]: This master's current role is: LEADER
02:07:55.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.078824 19831 sys_catalog.cc:458] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9 [sys.catalog]: This master's current role is: FOLLOWER
02:07:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.110915 19843 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.122488 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/logs
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data/info.pb
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:55.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.65:0
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.65
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.65
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:55.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:07:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.388651 19800 catalog_manager.cc:1261] Loaded cluster ID: 2faed0373fde44a7a181f5a262b47b0d
02:07:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.389357 19800 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: loading cluster ID for follower catalog manager: success
02:07:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.396862 19800 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: acquiring CA information for follower catalog manager: success
02:07:55.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.420478 19800 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 6619257b90774cf3acfa543fbf92f6cd: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:07:55.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.696688 19844 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:55.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.697443 19844 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:55.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.698359 19844 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:55.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.773168 19844 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:55.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.774757 19844 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.22.65
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.844534 19844 tablet_server_runner.cc:78] Tablet server non-default flags:
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.65:0
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data/info.pb
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.65
02:07:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.65
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/logs
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:55.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.846364 19844 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:55.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.849125 19844 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.880245 19850 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:56.129695 19843 catalog_manager.cc:1261] Loaded cluster ID: 2faed0373fde44a7a181f5a262b47b0d
02:07:56.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:56.130389 19843 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: loading cluster ID for follower catalog manager: success
02:07:56.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:56.147742 19843 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: acquiring CA information for follower catalog manager: success
02:07:56.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:56.155159 19843 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 17196fcbdd6c4beca1694c6a41db19b9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:07:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.893594 19851 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:57.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:55.893952 19854 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:57.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:55.894429 19844 server_base.cc:1047] running on GCE node
02:07:57.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.540500 19844 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:07:57.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.550884 19844 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:07:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.552699 19844 hybrid_clock.cc:648] HybridClock initialized: now 1756606077552663 us; error 51 us; skew 500 ppm
02:07:57.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.554090 19844 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:07:57.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.577268 19844 webserver.cc:480] Webserver started at http://127.19.22.65:40077/ using document root <none> and password file <none>
02:07:57.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.579061 19844 fs_manager.cc:362] Metadata directory not provided
02:07:57.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.587198 19844 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:07:57.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.588330 19844 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:07:57.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.598137 19844 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data/instance:
02:07:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c442119916b446068b68cacd59b591f3"
02:07:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:57 on dist-test-slave-kjk0"
02:07:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.600275 19844 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal/instance:
02:07:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c442119916b446068b68cacd59b591f3"
02:07:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:57 on dist-test-slave-kjk0"
02:07:57.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.618155 19844 fs_manager.cc:696] Time spent creating directory manager: real 0.017s user 0.009s sys 0.004s
02:07:57.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.632073 19861 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:07:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.633841 19844 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.001s sys 0.008s
02:07:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.634527 19844 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal
02:07:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c442119916b446068b68cacd59b591f3"
02:07:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:07:57 on dist-test-slave-kjk0"
02:07:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.635264 19844 fs_report.cc:389] FS layout report
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data/data
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:07:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:07:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.767931 19844 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:07:57.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.770448 19844 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:07:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.772150 19844 kserver.cc:163] Server-wide thread pool size limit: 3276
02:07:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.783597 19844 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:07:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.796716 19844 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:07:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.797649 19844 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
02:07:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.798265 19844 ts_tablet_manager.cc:610] Registered 0 tablets
02:07:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:57.798626 19844 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:07:58.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.049060 19844 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.65:39873
02:07:58.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.049181 19974 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.65:39873 every 8 connection(s)
02:07:58.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.052393 19844 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/data/info.pb
02:07:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.063712 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 19844
02:07:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.064544 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-0/wal/instance
02:07:58.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.091117 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/logs
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data/info.pb
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:07:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.66:0
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.66
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.66
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:07:58.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.183156 19975 heartbeater.cc:344] Connected to a master server at 127.19.22.124:38429
02:07:58.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.183696 19975 heartbeater.cc:461] Registering TS with master...
02:07:58.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.185071 19975 heartbeater.cc:507] Master 127.19.22.124:38429 requested a full tablet report, sending...
02:07:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.188483 19745 ts_manager.cc:194] Registered new tserver with Master: c442119916b446068b68cacd59b591f3 (127.19.22.65:39873)
02:07:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.188614 19976 heartbeater.cc:344] Connected to a master server at 127.19.22.125:33753
02:07:58.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.189002 19976 heartbeater.cc:461] Registering TS with master...
02:07:58.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.190068 19976 heartbeater.cc:507] Master 127.19.22.125:33753 requested a full tablet report, sending...
02:07:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.192653 19665 ts_manager.cc:194] Registered new tserver with Master: c442119916b446068b68cacd59b591f3 (127.19.22.65:39873)
02:07:58.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.195497 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.22.65:42013
02:07:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.201422 19980 heartbeater.cc:344] Connected to a master server at 127.19.22.126:46717
02:07:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.201828 19980 heartbeater.cc:461] Registering TS with master...
02:07:58.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.202541 19980 heartbeater.cc:507] Master 127.19.22.126:46717 requested a full tablet report, sending...
02:07:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.205526 19590 ts_manager.cc:194] Registered new tserver with Master: c442119916b446068b68cacd59b591f3 (127.19.22.65:39873)
02:07:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.493983 19979 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:07:58.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.494467 19979 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:07:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.494925 19979 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:07:58.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.527987 19979 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:07:58.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.528834 19979 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.22.66
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.565722 19979 tablet_server_runner.cc:78] Tablet server non-default flags:
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.66:0
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:07:58.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data/info.pb
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.66
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.66
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/logs
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:07:58.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:07:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:07:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.567178 19979 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:07:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:58.569448 19979 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:07:58.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.586521 19992 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:07:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:07:59.200552 19976 heartbeater.cc:499] Master 127.19.22.125:33753 was elected leader, sending a full tablet report...
02:08:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:58.587852 19993 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:07:59.989312 19991 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 19979
02:08:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.124320 19995 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.124643 19994 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1537 milliseconds
02:08:00.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.121801 19979 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.535s user 0.467s sys 0.867s
02:08:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.125784 19979 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.539s user 0.468s sys 0.868s
02:08:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.126287 19979 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:00.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.133241 19979 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:00.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.137388 19979 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:00.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.138752 19979 hybrid_clock.cc:648] HybridClock initialized: now 1756606080138713 us; error 43 us; skew 500 ppm
02:08:00.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.139631 19979 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.147189 19979 webserver.cc:480] Webserver started at http://127.19.22.66:34851/ using document root <none> and password file <none>
02:08:00.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.149413 19979 fs_manager.cc:362] Metadata directory not provided
02:08:00.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.149760 19979 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.150301 19979 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.154778 19979 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data/instance:
02:08:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82fa71729bad4dd586fe156a7177f6ba"
02:08:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:00 on dist-test-slave-kjk0"
02:08:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.155982 19979 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal/instance:
02:08:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82fa71729bad4dd586fe156a7177f6ba"
02:08:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:00 on dist-test-slave-kjk0"
02:08:00.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.163267 19979 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
02:08:00.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.169358 20002 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:00.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.170529 19979 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:08:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.170881 19979 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal
02:08:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "82fa71729bad4dd586fe156a7177f6ba"
02:08:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:00 on dist-test-slave-kjk0"
02:08:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.171257 19979 fs_report.cc:389] FS layout report
02:08:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data/data
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:00.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:00.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.232414 19979 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.234141 19979 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:00.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.234669 19979 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:00.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.237362 19979 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.241699 19979 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.241917 19979 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.242239 19979 ts_tablet_manager.cc:610] Registered 0 tablets
02:08:00.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.242434 19979 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:00.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.252765 19971 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1c0 after lost signal to thread 19845
02:08:00.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.254117 19971 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6140 after lost signal to thread 19974
02:08:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.422549 19979 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.66:46843
02:08:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.422610 20115 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.66:46843 every 8 connection(s)
02:08:00.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.426798 19979 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/data/info.pb
02:08:00.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.436239 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 19979
02:08:00.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.436843 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-1/wal/instance
02:08:00.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.476367 19545 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal
02:08:00.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/logs
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data/info.pb
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.67:0
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.67
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.67
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:00.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:08:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:08:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:00.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:08:00.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.481634 20118 heartbeater.cc:344] Connected to a master server at 127.19.22.126:46717
02:08:00.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.482265 20118 heartbeater.cc:461] Registering TS with master...
02:08:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.483738 20118 heartbeater.cc:507] Master 127.19.22.126:46717 requested a full tablet report, sending...
02:08:00.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.487377 19590 ts_manager.cc:194] Registered new tserver with Master: 82fa71729bad4dd586fe156a7177f6ba (127.19.22.66:46843)
02:08:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.495116 20117 heartbeater.cc:344] Connected to a master server at 127.19.22.125:33753
02:08:00.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.495554 20117 heartbeater.cc:461] Registering TS with master...
02:08:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.496564 20117 heartbeater.cc:507] Master 127.19.22.125:33753 requested a full tablet report, sending...
02:08:00.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.497117 20116 heartbeater.cc:344] Connected to a master server at 127.19.22.124:38429
02:08:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.497514 20116 heartbeater.cc:461] Registering TS with master...
02:08:00.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.498484 20116 heartbeater.cc:507] Master 127.19.22.124:38429 requested a full tablet report, sending...
02:08:00.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.500419 19665 ts_manager.cc:194] Registered new tserver with Master: 82fa71729bad4dd586fe156a7177f6ba (127.19.22.66:46843)
02:08:00.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.502691 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.22.66:48221
02:08:00.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.510079 19745 ts_manager.cc:194] Registered new tserver with Master: 82fa71729bad4dd586fe156a7177f6ba (127.19.22.66:46843)
02:08:00.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.812292 20126 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:00.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.812788 20126 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:00.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.813319 20126 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.864916 20126 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:00.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.866336 20126 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.22.67
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.903728 20126 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.22.84:44023
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.22.67:0
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data/info.pb
02:08:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.22.67
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.22.126:46717,127.19.22.125:33753,127.19.22.124:38429
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.22.67
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/logs
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:00.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.905179 20126 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:00.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:00.906957 20126 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:00.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.920356 20132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:01.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:01.508150 20117 heartbeater.cc:499] Master 127.19.22.125:33753 was elected leader, sending a full tablet report...
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.324504 20131 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 20126
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.996580 20131 kernel_stack_watchdog.cc:198] Thread 20126 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:02.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.997443 20126 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.076s user 0.639s sys 1.132s
02:08:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.997846 20126 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.077s user 0.639s sys 1.132s
02:08:02.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:00.922693 20133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:02.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.999130 20134 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2074 milliseconds
02:08:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:02.999909 20135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.000041 20126 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:03.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.004923 20126 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.007901 20126 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.009521 20126 hybrid_clock.cc:648] HybridClock initialized: now 1756606083009470 us; error 41 us; skew 500 ppm
02:08:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.010778 20126 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.018214 20126 webserver.cc:480] Webserver started at http://127.19.22.67:33279/ using document root <none> and password file <none>
02:08:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.019170 20126 fs_manager.cc:362] Metadata directory not provided
02:08:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.019368 20126 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.019763 20126 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:03.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.024210 20126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data/instance:
02:08:03.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab4768c9b69d4dd6b7b54a3fff72296e"
02:08:03.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:03 on dist-test-slave-kjk0"
02:08:03.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.025414 20126 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal/instance:
02:08:03.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab4768c9b69d4dd6b7b54a3fff72296e"
02:08:03.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:03 on dist-test-slave-kjk0"
02:08:03.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.033129 20126 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
02:08:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.039803 20143 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.040997 20126 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
02:08:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.041375 20126 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal
02:08:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab4768c9b69d4dd6b7b54a3fff72296e"
02:08:03.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:03 on dist-test-slave-kjk0"
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.041692 20126 fs_report.cc:389] FS layout report
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data/data
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.096441 20126 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:03.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.097959 20126 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:03.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.098376 20126 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.101053 20126 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:03.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.105456 20126 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.105706 20126 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.106014 20126 ts_tablet_manager.cc:610] Registered 0 tablets
02:08:03.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.106231 20126 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.256443 20126 rpc_server.cc:307] RPC server started. Bound to: 127.19.22.67:39337
02:08:03.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.256551 20255 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.22.67:39337 every 8 connection(s)
02:08:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.259284 20126 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/data/info.pb
02:08:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.259652 19545 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20126
02:08:03.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.259985 19545 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster4274240412319710758/ts-2/wal/instance
02:08:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.326889 20257 heartbeater.cc:344] Connected to a master server at 127.19.22.125:33753
02:08:03.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.327486 20257 heartbeater.cc:461] Registering TS with master...
02:08:03.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.328814 20257 heartbeater.cc:507] Master 127.19.22.125:33753 requested a full tablet report, sending...
02:08:03.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.329453 20256 heartbeater.cc:344] Connected to a master server at 127.19.22.124:38429
02:08:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.329820 20256 heartbeater.cc:461] Registering TS with master...
02:08:03.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.330698 20256 heartbeater.cc:507] Master 127.19.22.124:38429 requested a full tablet report, sending...
02:08:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.331722 20258 heartbeater.cc:344] Connected to a master server at 127.19.22.126:46717
02:08:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.331876 19665 ts_manager.cc:194] Registered new tserver with Master: ab4768c9b69d4dd6b7b54a3fff72296e (127.19.22.67:39337)
02:08:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.332077 20258 heartbeater.cc:461] Registering TS with master...
02:08:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.332741 20258 heartbeater.cc:507] Master 127.19.22.126:46717 requested a full tablet report, sending...
02:08:03.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.333365 19665 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.19.22.67:52749
02:08:03.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.334862 19745 ts_manager.cc:194] Registered new tserver with Master: ab4768c9b69d4dd6b7b54a3fff72296e (127.19.22.67:39337)
02:08:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.336575 19590 ts_manager.cc:194] Registered new tserver with Master: ab4768c9b69d4dd6b7b54a3fff72296e (127.19.22.67:39337)
02:08:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:03.352311 19545 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:08:03.363 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:08:03.365 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:03.381 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.22.126"
port: 46717
}
bound_http_address {
host: "127.19.22.126"
port: 32985
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.22.125"
port: 33753
}
bound_http_address {
host: "127.19.22.125"
port: 34583
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.22.124"
port: 38429
}
bound_http_address {
host: "127.19.22.124"
port: 36929
}
}
}
02:08:03.387 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:03.390 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.22.65"
port: 39873
}
bound_http_address {
host: "127.19.22.65"
port: 40077
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.22.66"
port: 46843
}
bound_http_address {
host: "127.19.22.66"
port: 34851
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.22.67"
port: 39337
}
bound_http_address {
host: "127.19.22.67"
port: 33279
}
}
}
02:08:03.391 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:08:03.597 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:08:03.623 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:08:03.734 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskX8607x/test-tmp
02:08:03.735 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:08:03.735 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskX8607x/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:08:03.737 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:08:04.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:04.063 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:08:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:04.067003 20267 mini_oidc.cc:150] Starting JWKS server
02:08:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:04.070825 20267 webserver.cc:480] Webserver started at http://127.0.0.1:46081/ using document root <none> and password file <none>
02:08:04.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:04.071398 20267 mini_oidc.cc:186] Starting OIDC Discovery server
02:08:04.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:04.073422 20267 webserver.cc:480] Webserver started at http://127.0.0.1:38047/ using document root <none> and password file <none>
02:08:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:08:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/principal' for realm 'KRBTEST.COM',
02:08:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:04 dist-test-slave-kjk0 krb5kdc[20279](info): setting up network...
02:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:04 dist-test-slave-kjk0 krb5kdc[20279](info): set up 2 sockets
02:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:04 dist-test-slave-kjk0 krb5kdc[20279](info): commencing operation
02:08:04.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:08:04.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:04.336815 20257 heartbeater.cc:499] Master 127.19.22.125:33753 was elected leader, sending a full tablet report...
02:08:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.176884 20267 mini_kdc.cc:121] Time spent starting KDC: real 2.066s user 0.000s sys 0.020s
02:08:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:08:06.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:06.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:08:06.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:08:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:06.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:08:06.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:08:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:08:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:08:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/test-user.keytab.
02:08:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/test-user.keytab.
02:08:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:06 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606086, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:06.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:08:06Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:08:06.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:08:06Z Disabled control of system clock
02:08:06.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.254@KRBTEST.COM; defaulting to no policy
02:08:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.254@KRBTEST.COM" created.
02:08:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:06.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:06.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.254@KRBTEST.COM; defaulting to no policy
02:08:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:06.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.254@KRBTEST.COM" created.
02:08:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:06.403062 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:06.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:06.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.727244 20297 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.727921 20297 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:06.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.728679 20297 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.761035 20297 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.761320 20297 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.761721 20297 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.762046 20297 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.762336 20297 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:06.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.762614 20297 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.762832 20297 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:06.802750 20297 master_runner.cc:386] Master server non-default flags:
02:08:06.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:06.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:06.804435 20297 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:06.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:06.806012 20297 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.818457 20305 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:07.506389 19776 debug-util.cc:398] Leaking SignalData structure 0x7b080006f060 after lost signal to thread 19707
02:08:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:07.507421 19776 debug-util.cc:398] Leaking SignalData structure 0x7b080003cce0 after lost signal to thread 19779
02:08:08.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.225395 20304 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 20297
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.749387 20304 kernel_stack_watchdog.cc:198] Thread 20297 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 403ms:
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:08.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:06.818922 20306 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:08.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.751500 20307 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1931 milliseconds
02:08:08.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.755571 20297 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.937s user 0.000s sys 0.006s
02:08:08.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.756017 20297 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.937s user 0.000s sys 0.006s
02:08:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:08.757583 20308 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:08.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.757596 20297 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.759480 20297 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:08.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.762807 20297 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:08.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.764338 20297 hybrid_clock.cc:648] HybridClock initialized: now 1756606088764270 us; error 82 us; skew 500 ppm
02:08:08.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.765216 20297 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:08.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:08 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.771584 20297 init.cc:377] Logged in from keytab as kudu/127.19.202.254@KRBTEST.COM (short username kudu)
02:08:08.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.777794 20297 webserver.cc:480] Webserver started at http://127.19.202.254:35171/ using document root <none> and password file <none>
02:08:08.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.778789 20297 fs_manager.cc:362] Metadata directory not provided
02:08:08.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.779004 20297 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:08.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.779474 20297 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.784226 20297 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/instance:
02:08:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:08 on dist-test-slave-kjk0"
02:08:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.785512 20297 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal/instance:
02:08:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:08 on dist-test-slave-kjk0"
02:08:08.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.793932 20297 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
02:08:08.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.800770 20322 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.802050 20297 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:08:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.802443 20297 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:08 on dist-test-slave-kjk0"
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.802810 20297 fs_report.cc:389] FS layout report
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/data
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:08.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.872471 20297 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.882694 20297 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.883232 20297 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.973090 20297 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.254:43435
02:08:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.973259 20373 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.254:43435 every 8 connection(s)
02:08:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.976143 20297 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:08.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.985472 20374 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.989468 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20297
02:08:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:08.990077 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal/instance
02:08:09.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.009090 20374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } has no permanent_uuid. Determining permanent_uuid...
02:08:09.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.253@KRBTEST.COM; defaulting to no policy
02:08:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.253@KRBTEST.COM" created.
02:08:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:09.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:09.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:09 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.253@KRBTEST.COM; defaulting to no policy
02:08:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.253@KRBTEST.COM" created.
02:08:09.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.082144 20374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
02:08:09.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.085857 20323 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.098991 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:09.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:09.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.110249 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:09.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.148612 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 1
02:08:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.155257 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.223709 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 2
02:08:09.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.227461 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.318845 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 3
02:08:09.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.323210 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.466687 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 4
02:08:09.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.472916 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.558540 20383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.559340 20383 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:09.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.560096 20383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.594542 20383 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.594826 20383 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.595223 20383 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:09.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.595535 20383 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:09.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.595856 20383 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:09.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.596144 20383 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:09.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.596364 20383 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.638329 20383 master_runner.cc:386] Master server non-default flags:
02:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:09.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.640192 20383 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.641739 20383 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.656061 20389 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:09.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:09.772454 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 5
02:08:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.781883 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:10.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:10.329499 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 6
02:08:10.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:10.340014 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.055226 20388 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 20383
02:08:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.092156 20383 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.437s user 0.548s sys 0.858s
02:08:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.092535 20383 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.438s user 0.548s sys 0.858s
02:08:11.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:09.657404 20390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.094745 20392 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.098780 20391 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1443 milliseconds
02:08:11.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.098794 20383 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:11.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.100159 20383 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.103622 20383 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.105047 20383 hybrid_clock.cc:648] HybridClock initialized: now 1756606091104958 us; error 62 us; skew 500 ppm
02:08:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.105957 20383 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:11.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:11 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:11.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.112406 20383 init.cc:377] Logged in from keytab as kudu/127.19.202.253@KRBTEST.COM (short username kudu)
02:08:11.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.117794 20383 webserver.cc:480] Webserver started at http://127.19.202.253:46831/ using document root <none> and password file <none>
02:08:11.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.118762 20383 fs_manager.cc:362] Metadata directory not provided
02:08:11.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.118960 20383 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:11.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.119346 20383 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.123759 20383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/instance:
02:08:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "034bbf011b9c4efc93529128399faa68"
02:08:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:11 on dist-test-slave-kjk0"
02:08:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.124858 20383 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal/instance:
02:08:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "034bbf011b9c4efc93529128399faa68"
02:08:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:11 on dist-test-slave-kjk0"
02:08:11.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.132128 20383 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
02:08:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.137849 20404 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.139042 20383 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
02:08:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.139365 20383 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "034bbf011b9c4efc93529128399faa68"
02:08:11.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:11 on dist-test-slave-kjk0"
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.139672 20383 fs_report.cc:389] FS layout report
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/data
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:11.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:11.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.194869 20383 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.200663 20383 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.201161 20383 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:11.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.284966 20383 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.253:46817
02:08:11.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.285070 20455 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.253:46817 every 8 connection(s)
02:08:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.287784 20383 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.293226 20456 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.294292 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20383
02:08:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.294700 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal/instance
02:08:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.304740 20456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.252@KRBTEST.COM; defaulting to no policy
02:08:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.252@KRBTEST.COM" created.
02:08:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:11 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.345101 20456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.252@KRBTEST.COM; defaulting to no policy
02:08:11.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.252@KRBTEST.COM" created.
02:08:11.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:11 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.370498 20456 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.372669 20405 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.376708 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.379812 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:11.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:11.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.400610 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } attempt: 7
02:08:11.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:11 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:11.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.422410 20374 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } has no permanent_uuid. Determining permanent_uuid...
02:08:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.426288 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 1
02:08:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.431691 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.435967 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.490257 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 1
02:08:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.494560 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.504381 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 2
02:08:11.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.508414 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.576071 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 2
02:08:11.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.579967 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.599936 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 3
02:08:11.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.604485 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.665441 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 3
02:08:11.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.670117 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.707587 20466 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.708197 20466 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.708885 20466 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.740823 20466 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.741094 20466 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.741475 20466 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.741701 20466 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.741909 20466 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.742097 20466 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.742239 20466 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.747893 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 4
02:08:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.752238 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.781401 20466 master_runner.cc:386] Master server non-default flags:
02:08:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.782902 20466 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:11.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.784317 20466 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:11.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.797962 20474 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:11.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:11.810633 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 4
02:08:11.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.816334 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:12.052441 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 5
02:08:12.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:12.065301 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:12.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:12.099984 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 5
02:08:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:12.108561 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:12.612908 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 6
02:08:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:12.623852 20456 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:12.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:12.661579 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 6
02:08:12.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:12.672426 20374 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.19.202.252:36351: Network error: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:11.798743 20475 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:13.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.149264 20476 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:08:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.151051 20477 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.154264 20466 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.355s user 0.550s sys 0.802s
02:08:13.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.154558 20466 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.355s user 0.550s sys 0.802s
02:08:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.154839 20466 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:13.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.155997 20466 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:13.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.158882 20466 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:13.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.160251 20466 hybrid_clock.cc:648] HybridClock initialized: now 1756606093160203 us; error 64 us; skew 500 ppm
02:08:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.161178 20466 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:13.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:13.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.169734 20466 init.cc:377] Logged in from keytab as kudu/127.19.202.252@KRBTEST.COM (short username kudu)
02:08:13.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.176153 20466 webserver.cc:480] Webserver started at http://127.19.202.252:39615/ using document root <none> and password file <none>
02:08:13.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.177492 20466 fs_manager.cc:362] Metadata directory not provided
02:08:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.177795 20466 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.178376 20466 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.185853 20466 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/instance:
02:08:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:13 on dist-test-slave-kjk0"
02:08:13.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.187562 20466 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal/instance:
02:08:13.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:13.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:13 on dist-test-slave-kjk0"
02:08:13.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.198263 20466 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
02:08:13.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.205725 20492 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.207049 20466 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:08:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.207422 20466 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:13.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:13 on dist-test-slave-kjk0"
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.207782 20466 fs_report.cc:389] FS layout report
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/data
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:13.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.309131 20466 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.314399 20466 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.314796 20466 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.389931 20466 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.252:36351
02:08:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.389954 20543 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.252:36351 every 8 connection(s)
02:08:13.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.392787 20466 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:13.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.397740 20544 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:08:13.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.401600 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20466
02:08:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.402005 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal/instance
02:08:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.411852 20544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } has no permanent_uuid. Determining permanent_uuid...
02:08:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606086, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:13.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606086, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.461594 20544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } has no permanent_uuid. Determining permanent_uuid...
02:08:13.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606086, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.485179 20544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } has no permanent_uuid. Determining permanent_uuid...
02:08:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606093, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.517550 20544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap starting.
02:08:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.524264 20544 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Neither blocks nor log segments found. Creating new log.
02:08:13.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.526016 20544 log.cc:826] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Log is configured to *not* fsync() on all Append() calls
02:08:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.531163 20544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: No bootstrap required, opened a new log
02:08:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.549787 20544 raft_consensus.cc:357] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.550714 20544 raft_consensus.cc:383] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.551004 20544 raft_consensus.cc:738] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5cff62c0ec44a80abb47b2b4738f998, State: Initialized, Role: FOLLOWER
02:08:13.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.551954 20544 consensus_queue.cc:260] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.554126 20555 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.554694 20555 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: FOLLOWER
02:08:13.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.555519 20544 sys_catalog.cc:564] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: configured and running, proceeding with master startup.
02:08:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.570535 20498 tablet.cc:2378] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.582631 20566 catalog_manager.cc:1560] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:08:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.582981 20566 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:08:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.578722 20466 master_runner.cc:186] Error getting master registration for 127.19.202.254:43435: OK, instance_id { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" instance_seqno: 1756606088948235 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.609040 20466 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:13.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.684396 20456 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 7
02:08:13.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606091, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.706054 20374 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } attempt: 7
02:08:13.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.718086 20456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap starting.
02:08:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:13 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606088, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:13.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.725715 20456 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Neither blocks nor log segments found. Creating new log.
02:08:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.728026 20456 log.cc:826] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Log is configured to *not* fsync() on all Append() calls
02:08:13.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.737336 20456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: No bootstrap required, opened a new log
02:08:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.742123 20374 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap starting.
02:08:13.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.750172 20374 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Neither blocks nor log segments found. Creating new log.
02:08:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.752550 20374 log.cc:826] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Log is configured to *not* fsync() on all Append() calls
02:08:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.756384 20456 raft_consensus.cc:357] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.757400 20456 raft_consensus.cc:383] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:13.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.757685 20456 raft_consensus.cc:738] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 034bbf011b9c4efc93529128399faa68, State: Initialized, Role: FOLLOWER
02:08:13.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.758662 20456 consensus_queue.cc:260] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.763685 20374 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: No bootstrap required, opened a new log
02:08:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.764859 20572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.765702 20572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: FOLLOWER
02:08:13.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.767225 20456 sys_catalog.cc:564] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: configured and running, proceeding with master startup.
02:08:13.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.771890 20572 raft_consensus.cc:491] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.772801 20572 raft_consensus.cc:513] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.779858 20519 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "034bbf011b9c4efc93529128399faa68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b5cff62c0ec44a80abb47b2b4738f998" is_pre_election: true
02:08:13.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.780964 20519 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 034bbf011b9c4efc93529128399faa68 in term 0.
02:08:13.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.783000 20405 leader_election.cc:304] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 034bbf011b9c4efc93529128399faa68, b5cff62c0ec44a80abb47b2b4738f998; no voters:
02:08:13.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.787472 20572 leader_election.cc:290] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), b5cff62c0ec44a80abb47b2b4738f998 (127.19.202.252:36351)
02:08:13.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.788601 20572 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:08:13.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.789711 20572 raft_consensus.cc:491] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:13.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.791610 20572 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 0 FOLLOWER]: Advancing to term 1
02:08:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.794229 20374 raft_consensus.cc:357] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.795300 20374 raft_consensus.cc:383] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:08:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.795756 20374 raft_consensus.cc:738] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d47520b5d9b348228a7ae8211f4a16d4, State: Initialized, Role: FOLLOWER
02:08:13.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.796927 20374 consensus_queue.cc:260] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.800912 20572 raft_consensus.cc:513] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.805856 20519 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "034bbf011b9c4efc93529128399faa68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.806612 20519 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 0 FOLLOWER]: Advancing to term 1
02:08:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.812779 20577 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.815164 20519 raft_consensus.cc:2466] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 034bbf011b9c4efc93529128399faa68 in term 1.
02:08:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.814409 20348 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "034bbf011b9c4efc93529128399faa68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.815759 20577 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.815963 20348 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 0 FOLLOWER]: Advancing to term 1
02:08:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.816218 20405 leader_election.cc:304] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 034bbf011b9c4efc93529128399faa68, b5cff62c0ec44a80abb47b2b4738f998; no voters:
02:08:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.816924 20572 leader_election.cc:290] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [CANDIDATE]: Term 1 election: Requested vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), b5cff62c0ec44a80abb47b2b4738f998 (127.19.202.252:36351)
02:08:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.817442 20572 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 FOLLOWER]: Leader election won for term 1
02:08:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.817873 20572 raft_consensus.cc:695] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 LEADER]: Becoming Leader. State: Replica: 034bbf011b9c4efc93529128399faa68, State: Running, Role: LEADER
02:08:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.818740 20572 consensus_queue.cc:237] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.815074 20349 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "034bbf011b9c4efc93529128399faa68" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4" is_pre_election: true
02:08:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.815531 20374 sys_catalog.cc:564] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: configured and running, proceeding with master startup.
02:08:13.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.829942 20348 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 034bbf011b9c4efc93529128399faa68 in term 1.
02:08:13.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.850553 20575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 034bbf011b9c4efc93529128399faa68. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:13.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.851984 20575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: LEADER
02:08:13.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.856447 20586 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:08:13.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.890044 20586 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:08:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.895288 20328 tablet.cc:2378] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:13.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.910113 20598 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:08:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.918725 20599 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:08:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.935130 20519 raft_consensus.cc:1273] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Refusing update from remote peer 034bbf011b9c4efc93529128399faa68: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.940265 20348 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Refusing update from remote peer 034bbf011b9c4efc93529128399faa68: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:08:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.945660 20575 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.947114 20575 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:08:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:13.959342 20410 tablet.cc:2378] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:08:13.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.919937 20297 master_runner.cc:186] Error getting master registration for 127.19.202.253:46817: OK, instance_id { permanent_uuid: "034bbf011b9c4efc93529128399faa68" instance_seqno: 1756606091264185 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:08:13.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:13.995349 20297 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.008888 20555 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 034bbf011b9c4efc93529128399faa68. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.010628 20555 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: FOLLOWER
02:08:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.026031 20600 mvcc.cc:204] Tried to move back new op lower bound from 7195058560742084608 to 7195058560293326848. Current Snapshot: MvccSnapshot[applied={T|T < 7195058560742084608}]
02:08:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.033471 20601 mvcc.cc:204] Tried to move back new op lower bound from 7195058560742084608 to 7195058560293326848. Current Snapshot: MvccSnapshot[applied={T|T < 7195058560742084608}]
02:08:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.035815 20577 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 034bbf011b9c4efc93529128399faa68. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.036746 20577 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.042641 20555 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.043404 20555 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: FOLLOWER
02:08:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.048961 20575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.049839 20575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: LEADER
02:08:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.053330 20602 mvcc.cc:204] Tried to move back new op lower bound from 7195058560742084608 to 7195058560293326848. Current Snapshot: MvccSnapshot[applied={T|T < 7195058560742084608}]
02:08:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.053555 20586 catalog_manager.cc:1349] Generated new cluster ID: 4fae66c10661427685a19797198cb005
02:08:14.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.053865 20586 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:08:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.054862 20575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.055773 20575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: LEADER
02:08:14.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.081808 20577 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "034bbf011b9c4efc93529128399faa68" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:14.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.082522 20577 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:14.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.099855 20586 catalog_manager.cc:1372] Generated new certificate authority record
02:08:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.102864 20586 catalog_manager.cc:1506] Loading token signing keys...
02:08:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.139945 20586 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Generated new TSK 0
02:08:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.141083 20586 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:08:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.193@KRBTEST.COM; defaulting to no policy
02:08:14.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.193@KRBTEST.COM" created.
02:08:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.193@KRBTEST.COM; defaulting to no policy
02:08:14.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.193@KRBTEST.COM" created.
02:08:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:14.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.326682 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/logs
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data/info.pb
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.193:0
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.193
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.193
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.193
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:14.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:14.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.589123 20566 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.589727 20566 catalog_manager.cc:1554] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: loading cluster ID for follower catalog manager: success
02:08:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.598551 20566 catalog_manager.cc:1576] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: acquiring CA information for follower catalog manager: success
02:08:14.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.604671 20566 catalog_manager.cc:1604] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.789230 20614 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.789983 20614 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.791150 20614 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:14.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.849432 20614 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.851126 20614 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.202.193
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.915935 20614 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.193
02:08:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.193:0
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data/info.pb
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.193
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.193
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/logs
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.917680 20614 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.919035 20614 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:14.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.949940 20621 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.961650 20598 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.963311 20598 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: loading cluster ID for follower catalog manager: success
02:08:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:14.987737 20598 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: acquiring CA information for follower catalog manager: success
02:08:15.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:15.019544 20598 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:17.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:16.341120 20620 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 20614
02:08:17.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:17.155647 20614 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.202s user 0.787s sys 1.222s
02:08:17.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:17.158833 20614 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.206s user 0.787s sys 1.222s
02:08:17.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:14.952894 20622 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:17.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.168788 20614 server_base.cc:1047] running on GCE node
02:08:17.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:17.172148 20629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.173718 20614 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.182122 20614 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.185750 20614 hybrid_clock.cc:648] HybridClock initialized: now 1756606097185332 us; error 408 us; skew 500 ppm
02:08:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.186885 20614 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:17.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:17 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606097, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.195868 20614 init.cc:377] Logged in from keytab as kudu/127.19.202.193@KRBTEST.COM (short username kudu)
02:08:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.229094 20614 webserver.cc:480] Webserver started at http://127.19.202.193:43525/ using document root <none> and password file <none>
02:08:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.230527 20614 fs_manager.cc:362] Metadata directory not provided
02:08:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.230826 20614 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.237500 20614 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:17.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.244921 20614 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data/instance:
02:08:17.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72ccd0a1ff544b2a9a0072a776ada85b"
02:08:17.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:17 on dist-test-slave-kjk0"
02:08:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.254854 20614 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal/instance:
02:08:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72ccd0a1ff544b2a9a0072a776ada85b"
02:08:17.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:17 on dist-test-slave-kjk0"
02:08:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.304422 20614 fs_manager.cc:696] Time spent creating directory manager: real 0.049s user 0.015s sys 0.015s
02:08:17.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.337935 20635 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.339442 20614 fs_manager.cc:730] Time spent opening block manager: real 0.023s user 0.007s sys 0.000s
02:08:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.339859 20614 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal
02:08:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72ccd0a1ff544b2a9a0072a776ada85b"
02:08:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:17 on dist-test-slave-kjk0"
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.340328 20614 fs_report.cc:389] FS layout report
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data/data
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:17.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.549043 20614 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.560432 20614 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:17.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.561064 20614 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.577692 20614 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.598992 20614 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.599320 20614 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.599619 20614 ts_tablet_manager.cc:610] Registered 0 tablets
02:08:17.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:17.599864 20614 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.015712 20614 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.193:46025
02:08:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.015847 20747 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.193:46025 every 8 connection(s)
02:08:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.019575 20614 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/data/info.pb
02:08:18.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.027197 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20614
02:08:18.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.027650 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-0/wal/instance
02:08:18.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.194@KRBTEST.COM; defaulting to no policy
02:08:18.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:18 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606097, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:18.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:18.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.194@KRBTEST.COM" created.
02:08:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:18 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606097, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:18.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:18 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606097, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:18.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.152367 20748 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.152887 20748 heartbeater.cc:461] Registering TS with master...
02:08:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.154268 20748 heartbeater.cc:507] Master 127.19.202.252:36351 requested a full tablet report, sending...
02:08:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.157754 20509 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.160995 20749 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.161371 20749 heartbeater.cc:461] Registering TS with master...
02:08:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.162326 20749 heartbeater.cc:507] Master 127.19.202.253:46817 requested a full tablet report, sending...
02:08:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.165429 20421 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:18.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.168145 20421 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:55939
02:08:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.175889 20756 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:18.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.176174 20756 heartbeater.cc:461] Registering TS with master...
02:08:18.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.176767 20756 heartbeater.cc:507] Master 127.19.202.254:43435 requested a full tablet report, sending...
02:08:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.179903 20338 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.182294 20267 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.19.202.194: real 0.141s user 0.005s sys 0.009s
02:08:18.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.194@KRBTEST.COM; defaulting to no policy
02:08:18.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:18.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.194@KRBTEST.COM" created.
02:08:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:18.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:18.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.234557 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/logs
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data/info.pb
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.194:0
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.194
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.194
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.194
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.627110 20763 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:18.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.627640 20763 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.628372 20763 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.661810 20763 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:18.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.662757 20763 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.202.194
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.701570 20763 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.194
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.194:0
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data/info.pb
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.194
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.194
02:08:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/logs
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:18.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.702986 20763 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:18.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.703970 20763 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:18.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.717293 20771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:18.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:18 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606098, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:18.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:18.777508 20318 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:19.174185 20749 heartbeater.cc:499] Master 127.19.202.253:46817 was elected leader, sending a full tablet report...
02:08:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.120383 20770 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 20763
02:08:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:18.718533 20772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.121209 20763 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.405s user 0.501s sys 0.861s
02:08:20.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.121614 20763 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.405s user 0.501s sys 0.861s
02:08:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.129420 20776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.131197 20763 server_base.cc:1047] running on GCE node
02:08:20.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.132297 20763 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.134411 20763 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.135778 20763 hybrid_clock.cc:648] HybridClock initialized: now 1756606100135729 us; error 46 us; skew 500 ppm
02:08:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.136602 20763 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:20.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:20 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606100, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.143115 20763 init.cc:377] Logged in from keytab as kudu/127.19.202.194@KRBTEST.COM (short username kudu)
02:08:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.148505 20763 webserver.cc:480] Webserver started at http://127.19.202.194:44525/ using document root <none> and password file <none>
02:08:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.149505 20763 fs_manager.cc:362] Metadata directory not provided
02:08:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.149735 20763 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.150137 20763 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.154696 20763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data/instance:
02:08:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ed97577870d4e989bfaea64821d71d6"
02:08:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:20 on dist-test-slave-kjk0"
02:08:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.155817 20763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal/instance:
02:08:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ed97577870d4e989bfaea64821d71d6"
02:08:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:20 on dist-test-slave-kjk0"
02:08:20.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.163367 20763 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
02:08:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.169761 20783 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.171144 20763 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.171554 20763 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ed97577870d4e989bfaea64821d71d6"
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:20 on dist-test-slave-kjk0"
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.171983 20763 fs_report.cc:389] FS layout report
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data/data
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:20.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:20.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.260151 20763 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:20.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.268045 20763 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:20.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.268616 20763 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:20.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.272239 20763 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.278645 20763 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.278885 20763 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.279214 20763 ts_tablet_manager.cc:610] Registered 0 tablets
02:08:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.279474 20763 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:20.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.454541 20763 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.194:33553
02:08:20.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.454653 20895 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.194:33553 every 8 connection(s)
02:08:20.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.457396 20763 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/data/info.pb
02:08:20.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.461730 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20763
02:08:20.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.462345 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-1/wal/instance
02:08:20.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:20 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606100, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.19.202.195@KRBTEST.COM; defaulting to no policy
02:08:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:20 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606100, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.19.202.195@KRBTEST.COM" created.
02:08:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:20 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606100, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:20.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.528339 20898 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.528715 20896 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.528905 20898 heartbeater.cc:461] Registering TS with master...
02:08:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.529156 20896 heartbeater.cc:461] Registering TS with master...
02:08:20.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.530450 20898 heartbeater.cc:507] Master 127.19.202.254:43435 requested a full tablet report, sending...
02:08:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.530468 20896 heartbeater.cc:507] Master 127.19.202.252:36351 requested a full tablet report, sending...
02:08:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.531948 20897 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.532292 20897 heartbeater.cc:461] Registering TS with master...
02:08:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.533202 20897 heartbeater.cc:507] Master 127.19.202.253:46817 requested a full tablet report, sending...
02:08:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.534062 20338 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.534268 20509 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:20.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.535463 20421 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.537526 20421 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:42775
02:08:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.19.202.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:20.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.19.202.195@KRBTEST.COM; defaulting to no policy
02:08:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:20.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.19.202.195@KRBTEST.COM" created.
02:08:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:08:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:20.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.19.202.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab.
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.606551 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/logs
02:08:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data/info.pb
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.195:0
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.195
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.195
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.195
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.917407 20911 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.917953 20911 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.918862 20911 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:20.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.954368 20911 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:20.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:20.955427 20911 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.202.195
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.993746 20911 tablet_server_runner.cc:78] Tablet server non-default flags:
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.195
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.195:0
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data/info.pb
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.195
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:20.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.202.195
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/logs
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:20.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.995456 20911 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:20.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:20.996680 20911 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:21.010532 20918 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:21 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606101, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:21.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:21.118559 20400 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:21.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:21.540617 20897 heartbeater.cc:499] Master 127.19.202.253:46817 was elected leader, sending a full tablet report...
02:08:21.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:21.599133 19621 debug-util.cc:398] Leaking SignalData structure 0x7b0800093de0 after lost signal to thread 19558
02:08:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:21.600069 19621 debug-util.cc:398] Leaking SignalData structure 0x7b080009dfc0 after lost signal to thread 19624
02:08:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:22.416002 20917 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 20911
02:08:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:23.099740 20911 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.089s user 0.650s sys 1.105s
02:08:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:23.100102 20911 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.089s user 0.650s sys 1.105s
02:08:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:21.010912 20919 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:23.101727 20920 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2088 milliseconds
02:08:23.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:23.102739 20921 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:23.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.102669 20911 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:23.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.106524 20911 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.109244 20911 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.110682 20911 hybrid_clock.cc:648] HybridClock initialized: now 1756606103110593 us; error 95 us; skew 500 ppm
02:08:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.111897 20911 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.120609 20911 init.cc:377] Logged in from keytab as kudu/127.19.202.195@KRBTEST.COM (short username kudu)
02:08:23.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.127502 20911 webserver.cc:480] Webserver started at http://127.19.202.195:45273/ using document root <none> and password file <none>
02:08:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.128815 20911 fs_manager.cc:362] Metadata directory not provided
02:08:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.129149 20911 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.129783 20911 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
02:08:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.137100 20911 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data/instance:
02:08:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "077b681b638f472096f34e72869e5de4"
02:08:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:23 on dist-test-slave-kjk0"
02:08:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.138816 20911 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal/instance:
02:08:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "077b681b638f472096f34e72869e5de4"
02:08:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:23 on dist-test-slave-kjk0"
02:08:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.148952 20911 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
02:08:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.156663 20934 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.158053 20911 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:08:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.158489 20911 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal
02:08:23.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "077b681b638f472096f34e72869e5de4"
02:08:23.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:23 on dist-test-slave-kjk0"
02:08:23.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.159060 20911 fs_report.cc:389] FS layout report
02:08:23.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data/data
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:23.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:23.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.175227 20488 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:23.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.257964 20911 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:23.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.266415 20911 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:23.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.267040 20911 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.270680 20911 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:08:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.277158 20911 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:08:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.277470 20911 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:08:23.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.277772 20911 ts_tablet_manager.cc:610] Registered 0 tablets
02:08:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.277997 20911 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:08:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.496678 20911 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.195:39229
02:08:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.496796 21046 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.195:39229 every 8 connection(s)
02:08:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.499358 20911 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/data/info.pb
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.502665 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 20911
02:08:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.503158 20267 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/ts-2/wal/instance
02:08:23.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.574126 21047 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.574726 21047 heartbeater.cc:461] Registering TS with master...
02:08:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.576529 21047 heartbeater.cc:507] Master 127.19.202.252:36351 requested a full tablet report, sending...
02:08:23.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.581553 20509 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:23.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:23.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.606328 21049 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:23.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.606756 21049 heartbeater.cc:461] Registering TS with master...
02:08:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.607848 21049 heartbeater.cc:507] Master 127.19.202.254:43435 requested a full tablet report, sending...
02:08:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.608235 21048 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.608623 21048 heartbeater.cc:461] Registering TS with master...
02:08:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.609726 21048 heartbeater.cc:507] Master 127.19.202.253:46817 requested a full tablet report, sending...
02:08:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.610301 20338 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.612350 20420 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.614257 20420 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:50729
02:08:23.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.621209 20267 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
02:08:23.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:08:23.652 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:08:23.653 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:08:23.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.202.254"
port: 43435
}
bound_http_address {
host: "127.19.202.254"
port: 35171
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.202.253"
port: 46817
}
bound_http_address {
host: "127.19.202.253"
port: 46831
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.202.252"
port: 36351
}
bound_http_address {
host: "127.19.202.252"
port: 39615
}
}
}
02:08:23.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:08:23.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.202.193"
port: 46025
}
bound_http_address {
host: "127.19.202.193"
port: 43525
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.202.194"
port: 33553
}
bound_http_address {
host: "127.19.202.194"
port: 44525
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.202.195"
port: 39229
}
bound_http_address {
host: "127.19.202.195"
port: 45273
}
}
}
02:08:23.659 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:08:23.661 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:08:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:23.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:23.697 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.700 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc
02:08:23.708 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:23.709 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc
02:08:23.711 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:23.715 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:08:23.718 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:08:23.718 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:08:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:23 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:23.763 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.763 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc
02:08:23.765 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:23.766 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.252:36351
02:08:23.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:08:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.768883 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20466
02:08:23.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.805 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.254:43435
02:08:23.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:08:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.806110 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20297
02:08:23.837 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.837 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.253:46817
02:08:23.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:08:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.838375 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20383
02:08:23.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:23.876 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.252:36351
02:08:23.880 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:23.882963 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39615
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:23.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.194159 20748 heartbeater.cc:646] Failed to heartbeat to 127.19.202.252:36351 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.233886 21061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:24.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.234697 21061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:24.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.235764 21061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.280414 21061 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:24.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.280668 21061 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.280938 21061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.281203 21061 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.281407 21061 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:24.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.281651 21061 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.281870 21061 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:24.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:24.323531 21061 master_runner.cc:386] Master server non-default flags:
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:24.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39615
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:24.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:24.324829 21061 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:24.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:24.325902 21061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:24.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.338125 21070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.566351 20896 heartbeater.cc:646] Failed to heartbeat to 127.19.202.252:36351 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.589855 21047 heartbeater.cc:646] Failed to heartbeat to 127.19.202.252:36351 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.19.202.252:36351: connect: Connection refused (error 111)
02:08:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:24.618829 21048 heartbeater.cc:499] Master 127.19.202.253:46817 was elected leader, sending a full tablet report...
02:08:25.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:24.338917 21071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:25.666571 21061 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.330s user 0.532s sys 0.795s
02:08:25.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:25.666885 21061 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.331s user 0.532s sys 0.795s
02:08:25.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.669631 21061 server_base.cc:1047] running on GCE node
02:08:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:25.671133 21084 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:25.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.672308 21061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:25.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.675230 21061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:25.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.676594 21061 hybrid_clock.cc:648] HybridClock initialized: now 1756606105676549 us; error 66 us; skew 500 ppm
02:08:25.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.677512 21061 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:25 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606105, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:25.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.685060 21061 init.cc:377] Logged in from keytab as kudu/127.19.202.252@KRBTEST.COM (short username kudu)
02:08:25.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.690949 21061 webserver.cc:480] Webserver started at http://127.19.202.252:39615/ using document root <none> and password file <none>
02:08:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.691886 21061 fs_manager.cc:362] Metadata directory not provided
02:08:25.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.692102 21061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:25.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.701336 21061 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
02:08:25.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.706429 21090 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.707926 21061 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:08:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.708315 21061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:13 on dist-test-slave-kjk0"
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.710508 21061 fs_report.cc:389] FS layout report
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/data
02:08:25.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:25.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.788307 21061 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:25.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.801343 21061 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.801865 21061 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:25.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.879855 21061 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.252:36351
02:08:25.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.879877 21142 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.252:36351 every 8 connection(s)
02:08:25.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.882961 21061 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.893281 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21061
02:08:25.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.893550 21143 sys_catalog.cc:263] Verifying existing consensus state
02:08:25.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:25.894 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.254:43435
02:08:25.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.896275 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35171
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:25.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.902161 21143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap starting.
02:08:25.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.940989 21143 log.cc:826] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Log is configured to *not* fsync() on all Append() calls
02:08:25.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.957474 21143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.958319 21143 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap complete.
02:08:25.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.982332 21143 raft_consensus.cc:357] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:25.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.984858 21143 raft_consensus.cc:738] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5cff62c0ec44a80abb47b2b4738f998, State: Initialized, Role: FOLLOWER
02:08:25.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.985808 21143 consensus_queue.cc:260] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:25.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.988785 21147 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:25.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.989681 21147 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: FOLLOWER
02:08:25.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:25.990942 21143 sys_catalog.cc:564] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: configured and running, proceeding with master startup.
02:08:26.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.034529 21158 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.034839 21158 catalog_manager.cc:1554] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: loading cluster ID for follower catalog manager: success
02:08:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.043921 21158 catalog_manager.cc:1576] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: acquiring CA information for follower catalog manager: success
02:08:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.049464 21158 catalog_manager.cc:1604] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:26.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.028502 21061 master_runner.cc:186] Error getting master registration for 127.19.202.254:43435: Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111),
02:08:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.057761 21061 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:26.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.238224 20748 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:26.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.245299 21108 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "72ccd0a1ff544b2a9a0072a776ada85b" instance_seqno: 1756606097930860) as {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:36341; Asking this server to re-register.
02:08:26.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.247576 20748 heartbeater.cc:461] Registering TS with master...
02:08:26.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.251579 21108 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.291682 21145 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.292237 21145 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.292984 21145 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.327414 21145 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.327659 21145 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.327971 21145 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.328179 21145 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.328373 21145 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.328558 21145 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:26.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.328708 21145 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.369560 21145 master_runner.cc:386] Master server non-default flags:
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35171
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.371043 21145 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:26.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.372416 21145 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:26.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.388475 21169 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.599812 20896 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:26.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.603173 21108 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5ed97577870d4e989bfaea64821d71d6" instance_seqno: 1756606100410382) as {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:51459; Asking this server to re-register.
02:08:26.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.604769 20896 heartbeater.cc:461] Registering TS with master...
02:08:26.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.608256 21108 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.616963 21047 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.619897 21108 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "077b681b638f472096f34e72869e5de4" instance_seqno: 1756606103450472) as {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:49207; Asking this server to re-register.
02:08:26.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.622412 21047 heartbeater.cc:461] Registering TS with master...
02:08:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:26.625620 21108 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:27 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606107, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.208362 20631 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:27.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.266594 21178 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:27.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.267446 21178 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:27.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.290714 21178 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.297554 21092 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111)
02:08:27.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.323590 21092 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435): Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111)
02:08:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.329396 21092 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.330051 21092 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998; no voters: 034bbf011b9c4efc93529128399faa68, d47520b5d9b348228a7ae8211f4a16d4
02:08:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.331302 21178 raft_consensus.cc:2747] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:08:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.788766 21168 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21145
02:08:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.958763 21168 kernel_stack_watchdog.cc:198] Thread 21145 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:08:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:27.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:27.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:26.389535 21170 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:27.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.959867 21145 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.573s user 0.573s sys 0.918s
02:08:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.960213 21145 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.575s user 0.573s sys 0.918s
02:08:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.968760 21145 server_base.cc:1047] running on GCE node
02:08:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:27.969533 21186 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.970768 21145 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:27.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.973902 21145 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:27.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.975430 21145 hybrid_clock.cc:648] HybridClock initialized: now 1756606107975400 us; error 51 us; skew 500 ppm
02:08:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.976284 21145 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:27 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606107, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.982800 21145 init.cc:377] Logged in from keytab as kudu/127.19.202.254@KRBTEST.COM (short username kudu)
02:08:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.988759 21145 webserver.cc:480] Webserver started at http://127.19.202.254:35171/ using document root <none> and password file <none>
02:08:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.989730 21145 fs_manager.cc:362] Metadata directory not provided
02:08:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.989952 21145 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:27.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:27.999346 21145 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
02:08:28.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.014299 21192 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:28.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.015532 21145 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.002s sys 0.011s
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.015846 21145 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:08 on dist-test-slave-kjk0"
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.017966 21145 fs_report.cc:389] FS layout report
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/data
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:28.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.073750 21145 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:28.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.081349 21145 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:28.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.081920 21145 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:28.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.171049 21145 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.254:43435
02:08:28.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.171067 21245 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.254:43435 every 8 connection(s)
02:08:28.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.175168 21145 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:28.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.180210 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21145
02:08:28.181 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:28.181 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.253:46817
02:08:28.182 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:08:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.184341 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46831
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:28.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:28.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:28.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.190560 21246 sys_catalog.cc:263] Verifying existing consensus state
02:08:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.196672 21246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap starting.
02:08:28.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.245349 21246 log.cc:826] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Log is configured to *not* fsync() on all Append() calls
02:08:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.292079 21246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:28.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.293192 21246 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap complete.
02:08:28.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:28 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606107, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.334520 21246 raft_consensus.cc:357] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.338104 21246 raft_consensus.cc:738] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d47520b5d9b348228a7ae8211f4a16d4, State: Initialized, Role: FOLLOWER
02:08:28.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.339322 21246 consensus_queue.cc:260] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.351080 21246 sys_catalog.cc:564] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: configured and running, proceeding with master startup.
02:08:28.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.348131 21253 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:28.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.354661 21253 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:28.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.359054 20756 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:28.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.413952 21264 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:28.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:28 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606107, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:28.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.414254 21264 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: loading cluster ID for follower catalog manager: success
02:08:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.454630 21264 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: acquiring CA information for follower catalog manager: success
02:08:28.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.467913 21264 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.468835 21145 master_runner.cc:186] Error getting master registration for 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111),
02:08:28.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.528374 21145 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:28.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.679158 20898 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:28.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.688342 21211 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5ed97577870d4e989bfaea64821d71d6" instance_seqno: 1756606100410382) as {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:55767; Asking this server to re-register.
02:08:28.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.689857 20898 heartbeater.cc:461] Registering TS with master...
02:08:28.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.693701 21211 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.701608 21049 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.704331 21211 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "077b681b638f472096f34e72869e5de4" instance_seqno: 1756606103450472) as {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:56003; Asking this server to re-register.
02:08:28.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.705513 21049 heartbeater.cc:461] Registering TS with master...
02:08:28.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.708963 21211 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:28.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.752925 21248 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:28.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.753787 21248 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:28.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.754920 21248 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.793561 21248 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:28.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.793846 21248 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:28.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.794236 21248 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:28.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.794548 21248 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.794845 21248 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:28.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.795177 21248 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:28.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.795421 21248 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.840831 21248 master_runner.cc:386] Master server non-default flags:
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46831
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.853471 21248 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:28.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:28.855048 21248 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.893481 21278 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.109305 21282 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.110250 21282 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:29.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.124550 21282 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.162981 21221 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b5cff62c0ec44a80abb47b2b4738f998" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4" is_pre_election: true
02:08:29.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.163933 21221 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5cff62c0ec44a80abb47b2b4738f998 in term 1.
02:08:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:29.164558 21092 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:29.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.167584 21092 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998, d47520b5d9b348228a7ae8211f4a16d4; no voters: 034bbf011b9c4efc93529128399faa68
02:08:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.168215 21282 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Leader pre-election won for term 2
02:08:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.168589 21282 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.169046 21282 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 1 FOLLOWER]: Advancing to term 2
02:08:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.189635 21282 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.193603 21221 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b5cff62c0ec44a80abb47b2b4738f998" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.193902 21282 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 election: Requested vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:29.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.194206 21221 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 1 FOLLOWER]: Advancing to term 2
02:08:29.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:29.203771 21092 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.220692 21221 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5cff62c0ec44a80abb47b2b4738f998 in term 2.
02:08:29.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.221971 21092 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998, d47520b5d9b348228a7ae8211f4a16d4; no voters: 034bbf011b9c4efc93529128399faa68
02:08:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.223058 21282 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Leader election won for term 2
02:08:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.245380 21282 raft_consensus.cc:695] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 LEADER]: Becoming Leader. State: Replica: b5cff62c0ec44a80abb47b2b4738f998, State: Running, Role: LEADER
02:08:29.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.246472 21282 consensus_queue.cc:237] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.278419 21286 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5cff62c0ec44a80abb47b2b4738f998. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.279697 21286 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: LEADER
02:08:29.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.294752 21288 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:08:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.330108 21288 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:08:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.352402 21288 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.352846 21288 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:08:29.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.367197 21211 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "72ccd0a1ff544b2a9a0072a776ada85b" instance_seqno: 1756606097930860) as {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:52885; Asking this server to re-register.
02:08:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.368768 20756 heartbeater.cc:461] Registering TS with master...
02:08:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.371699 21211 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.394018 21288 catalog_manager.cc:1506] Loading token signing keys...
02:08:29.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.406770 21288 catalog_manager.cc:5966] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Loaded TSK: 0
02:08:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.409186 21288 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:08:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.751976 21221 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Refusing update from remote peer b5cff62c0ec44a80abb47b2b4738f998: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:08:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.755719 21286 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
02:08:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:29.757659 21092 consensus_peers.cc:489] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 -> Peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Couldn't send request to peer 034bbf011b9c4efc93529128399faa68. Status: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:08:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.785648 21294 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5cff62c0ec44a80abb47b2b4738f998. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.786967 21294 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.801250 21294 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.802004 21294 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:29.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.804991 21286 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:29.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:29.805691 21286 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: LEADER
02:08:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:30 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606110, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:30.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.148351 20779 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:30.281213 21277 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21248
02:08:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:30.514660 21277 kernel_stack_watchdog.cc:198] Thread 21248 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 378ms:
02:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:28.913307 21279 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:30.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:30.516392 21248 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.615s user 0.478s sys 0.737s
02:08:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:30.516839 21248 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.615s user 0.478s sys 0.737s
02:08:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.523876 21248 server_base.cc:1047] running on GCE node
02:08:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:30.524531 21302 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:30.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.526024 21248 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:30.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.530566 21248 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.531950 21248 hybrid_clock.cc:648] HybridClock initialized: now 1756606110531897 us; error 63 us; skew 500 ppm
02:08:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.532804 21248 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:30.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:30 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606110, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:30.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.539333 21248 init.cc:377] Logged in from keytab as kudu/127.19.202.253@KRBTEST.COM (short username kudu)
02:08:30.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.545286 21248 webserver.cc:480] Webserver started at http://127.19.202.253:46831/ using document root <none> and password file <none>
02:08:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.546253 21248 fs_manager.cc:362] Metadata directory not provided
02:08:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.546499 21248 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.555383 21248 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.005s
02:08:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.560927 21308 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:30.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.562258 21248 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
02:08:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.562681 21248 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "034bbf011b9c4efc93529128399faa68"
02:08:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:11 on dist-test-slave-kjk0"
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.565527 21248 fs_report.cc:389] FS layout report
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/data
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:30.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.631800 21248 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.638077 21248 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.638574 21248 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:30.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.639779 20896 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.726912 21047 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:30.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.740581 21248 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.253:46817
02:08:30.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.740584 21361 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.253:46817 every 8 connection(s)
02:08:30.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.744596 21248 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:30.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.752739 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21248
02:08:30.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.760537 21362 sys_catalog.cc:263] Verifying existing consensus state
02:08:30.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.768733 21362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap starting.
02:08:30.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.818400 21362 log.cc:826] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Log is configured to *not* fsync() on all Append() calls
02:08:30.843 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.202.252:36351(127.19.202.252:36351)] connecting to peer
02:08:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.853236 21362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.854278 21362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap complete.
02:08:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.904824 21362 raft_consensus.cc:357] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:30.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.908934 21362 raft_consensus.cc:738] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 034bbf011b9c4efc93529128399faa68, State: Initialized, Role: FOLLOWER
02:08:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.910310 21362 consensus_queue.cc:260] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.937691 21368 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:30.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.939322 21368 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: FOLLOWER
02:08:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:30.939565 21362 sys_catalog.cc:564] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: configured and running, proceeding with master startup.
02:08:30.951 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.202.254:43435(127.19.202.254:43435)] connecting to peer
02:08:30.954 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.202.253:46817(127.19.202.253:46817)] connecting to peer
02:08:31.011 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.202.254:43435(127.19.202.254:43435)] Successfully connected to peer
02:08:31.024 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.202.252:36351(127.19.202.252:36351)] Successfully connected to peer
02:08:31.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.034293 21381 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.034592 21381 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: loading cluster ID for follower catalog manager: success
02:08:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.043016 21381 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: acquiring CA information for follower catalog manager: success
02:08:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.048538 21381 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:31.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:31 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606110, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:31.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.100157 21061 master_runner.cc:232] Existing masters have differing terms: 2 vs 1
02:08:31.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.100728 21061 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:31.156 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.19.202.253:46817(127.19.202.253:46817)] Successfully connected to peer
02:08:31.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.167677 21336 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 1 FOLLOWER]: Advancing to term 2
02:08:31.209 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:08:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.218292 21393 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.219384 21393 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: FOLLOWER
02:08:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.236502 21368 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5cff62c0ec44a80abb47b2b4738f998. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.237277 21368 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: FOLLOWER
02:08:31.252 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.202.254
02:08:31.255 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.202.252
02:08:31.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.335778 20748 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.385390 20749 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:31.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.398094 21325 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "72ccd0a1ff544b2a9a0072a776ada85b" instance_seqno: 1756606097930860) as {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:55415; Asking this server to re-register.
02:08:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.399713 20749 heartbeater.cc:461] Registering TS with master...
02:08:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.403079 21325 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:31.444 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.202.253
02:08:31.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:31 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:31.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:31 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.252@KRBTEST.COM
02:08:31.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.709285 20897 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:31.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.712682 21325 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5ed97577870d4e989bfaea64821d71d6" instance_seqno: 1756606100410382) as {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:46501; Asking this server to re-register.
02:08:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.715847 20897 heartbeater.cc:461] Registering TS with master...
02:08:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.719300 21325 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:31.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:31 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606103, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.756834 21395 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:31.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.757936 21395 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: LEADER
02:08:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.782032 21048 heartbeater.cc:344] Connected to a master server at 127.19.202.253:46817
02:08:31.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.782603 21048 heartbeater.cc:499] Master 127.19.202.253:46817 was elected leader, sending a full tablet report...
02:08:31.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.785211 21325 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "077b681b638f472096f34e72869e5de4" instance_seqno: 1756606103450472) as {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:43543; Asking this server to re-register.
02:08:31.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.786970 21048 heartbeater.cc:461] Registering TS with master...
02:08:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:31.792027 21325 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:31.840 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7c0a0976, L:/127.0.0.1:40260 - R:/127.19.202.254:43435] using SASL/GSSAPI
02:08:31.841 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xca2a1233, L:/127.0.0.1:43834 - R:/127.19.202.252:36351] using SASL/GSSAPI
02:08:31.858 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xda59122a, L:/127.0.0.1:54350 - R:/127.19.202.253:46817] using SASL/GSSAPI
02:08:31.968 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
02:08:31.969 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1211), Traces: [0ms] refreshing cache from master, [51ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.202.252:36351, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.202.254:43435, [184ms] Sub RPC ConnectToMaster: sending RPC to server master-127.19.202.253:46817, [1136ms] Sub RPC ConnectToMaster: received response from server master-127.19.202.252:36351: OK, [1135ms] Sub RPC ConnectToMaster: received response from server master-127.19.202.254:43435: OK, deferred=null) after lookup
02:08:33.003 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:08:33.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:08:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:33 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606113, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:33.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:08:33.034 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:33.034 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc
02:08:33.036 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:08:33.036 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.252:36351
02:08:33.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:08:33.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.037349 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21061
02:08:33.074 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:33.075 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.254:43435
02:08:33.075 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:08:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.075656 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21145
02:08:33.078 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.202.252:36351(127.19.202.252:36351)] handling channelInactive
02:08:33.079 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.202.252:36351(127.19.202.252:36351)] cleaning up while in state READY due to: connection closed
02:08:33.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.106784 20744 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab40 after lost signal to thread 20615
02:08:33.108 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.202.254:43435(127.19.202.254:43435)] handling channelInactive
02:08:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.107947 20744 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad1a0 after lost signal to thread 20747
02:08:33.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.202.254:43435(127.19.202.254:43435)] cleaning up while in state READY due to: connection closed
02:08:33.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:33.111 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.19.202.253:46817
02:08:33.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:08:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.111521 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21248
02:08:33.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:33 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606113, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.126817 20930 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:33.142 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.19.202.253:46817(127.19.202.253:46817)] handling channelInactive
02:08:33.142 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.19.202.253:46817(127.19.202.253:46817)] cleaning up while in state READY due to: connection closed
02:08:33.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:33.143 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.252:36351
02:08:33.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.145896 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:33.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39615
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.518702 21405 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:33.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.519343 21405 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:33.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.520128 21405 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.552670 21405 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.553030 21405 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.553426 21405 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:33.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.553746 21405 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.554049 21405 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.554337 21405 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.554580 21405 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:33.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.594516 21405 master_runner.cc:386] Master server non-default flags:
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.252
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.252:36351
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:33.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.252
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39615
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/logs
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.596243 21405 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:33.597862 21405 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:33.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.611232 21412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:35.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.016233 21411 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21405
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.705485 21411 kernel_stack_watchdog.cc:198] Thread 21405 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:35.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:33.612444 21413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.708900 21415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:35.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.709549 21405 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.097s user 0.706s sys 1.179s
02:08:35.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.709895 21405 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.097s user 0.706s sys 1.179s
02:08:35.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:35.715559 21414 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2100 milliseconds
02:08:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.715998 21405 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.717435 21405 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.720403 21405 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.721881 21405 hybrid_clock.cc:648] HybridClock initialized: now 1756606115721815 us; error 57 us; skew 500 ppm
02:08:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.723047 21405 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:35.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:35 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606115, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:35.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.730422 21405 init.cc:377] Logged in from keytab as kudu/127.19.202.252@KRBTEST.COM (short username kudu)
02:08:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.736776 21405 webserver.cc:480] Webserver started at http://127.19.202.252:39615/ using document root <none> and password file <none>
02:08:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.738180 21405 fs_manager.cc:362] Metadata directory not provided
02:08:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.738489 21405 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.750809 21405 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.004s
02:08:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.757246 21438 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.759897 21405 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.002s
02:08:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.760344 21405 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b5cff62c0ec44a80abb47b2b4738f998"
02:08:35.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:13 on dist-test-slave-kjk0"
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.763375 21405 fs_report.cc:389] FS layout report
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/wal
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/data
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:35.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.827350 21405 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.833420 21405 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:35.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.833958 21405 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.950734 21405 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.252:36351
02:08:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.950881 21491 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.252:36351 every 8 connection(s)
02:08:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.953711 21405 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-2/data/info.pb
02:08:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.963726 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21405
02:08:35.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:35.964 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.254:43435
02:08:35.964 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:08:35.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.965133 21492 sys_catalog.cc:263] Verifying existing consensus state
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.966084 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35171
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:35.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:35.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:35.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:35.972007 21492 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap starting.
02:08:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.012660 21492 log.cc:826] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Log is configured to *not* fsync() on all Append() calls
02:08:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.029218 21492 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:36.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.030076 21492 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Bootstrap complete.
02:08:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.063759 21492 raft_consensus.cc:357] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.066260 21492 raft_consensus.cc:738] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b5cff62c0ec44a80abb47b2b4738f998, State: Initialized, Role: FOLLOWER
02:08:36.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.067287 21492 consensus_queue.cc:260] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:36.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.070225 21496 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:36.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.071619 21496 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: FOLLOWER
02:08:36.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.072297 21492 sys_catalog.cc:564] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: configured and running, proceeding with master startup.
02:08:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.104998 21507 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.105584 21507 catalog_manager.cc:1554] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: loading cluster ID for follower catalog manager: success
02:08:36.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.114565 21507 catalog_manager.cc:1576] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: acquiring CA information for follower catalog manager: success
02:08:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.120589 21507 catalog_manager.cc:1604] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.107019 21405 master_runner.cc:186] Error getting master registration for 127.19.202.254:43435: Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111),
02:08:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.133273 21405 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:36.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.331326 21494 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:36.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.332036 21494 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:36.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.332792 21494 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:36.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.367715 21494 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:36.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.368028 21494 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:36.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.368427 21494 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:36.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.368755 21494 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.369104 21494 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.369401 21494 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.369642 21494 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.429994 21494 master_runner.cc:386] Master server non-default flags:
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.254
02:08:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.254:43435
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.254
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35171
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/logs
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.431742 21494 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.432417 20748 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.433125 21494 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:36.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.438840 21457 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "72ccd0a1ff544b2a9a0072a776ada85b" instance_seqno: 1756606097930860) as {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:35449; Asking this server to re-register.
02:08:36.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.440521 20748 heartbeater.cc:461] Registering TS with master...
02:08:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.444406 21457 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.475632 21518 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.725894 20896 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.729274 21457 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5ed97577870d4e989bfaea64821d71d6" instance_seqno: 1756606100410382) as {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:49381; Asking this server to re-register.
02:08:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.732203 20896 heartbeater.cc:461] Registering TS with master...
02:08:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.735978 21457 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.911754 21047 heartbeater.cc:344] Connected to a master server at 127.19.202.252:36351
02:08:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.915556 21457 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "077b681b638f472096f34e72869e5de4" instance_seqno: 1756606103450472) as {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:52343; Asking this server to re-register.
02:08:36.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.917946 21047 heartbeater.cc:461] Registering TS with master...
02:08:36.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:36.921979 21457 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:37.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.205655 21527 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:37.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.206290 21527 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:37 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606117, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.214018 20631 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:37.228607 21439 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111)
02:08:37.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.232760 21527 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:37.246187 21439 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435): Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111)
02:08:37.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:37.248467 21439 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:37.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.248910 21439 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998; no voters: 034bbf011b9c4efc93529128399faa68, d47520b5d9b348228a7ae8211f4a16d4
02:08:37.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:37.249732 21527 raft_consensus.cc:2747] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:37.860833 21517 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21494
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:38.937566 21517 kernel_stack_watchdog.cc:198] Thread 21494 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 381ms:
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:38.938268 21494 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.461s user 0.714s sys 1.424s
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:38.938697 21494 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.461s user 0.714s sys 1.424s
02:08:38.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:36.478065 21519 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:38.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:38.940177 21521 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:38.943783 21520 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2462 milliseconds
02:08:38.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.943859 21494 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:38.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.945432 21494 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:38.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.962536 21494 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.963980 21494 hybrid_clock.cc:648] HybridClock initialized: now 1756606118963932 us; error 74 us; skew 500 ppm
02:08:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.965047 21494 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:38.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:38 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606118, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:38.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.971515 21494 init.cc:377] Logged in from keytab as kudu/127.19.202.254@KRBTEST.COM (short username kudu)
02:08:38.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.991132 21494 webserver.cc:480] Webserver started at http://127.19.202.254:35171/ using document root <none> and password file <none>
02:08:38.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.992442 21494 fs_manager.cc:362] Metadata directory not provided
02:08:38.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:38.992769 21494 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.004592 21494 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
02:08:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.010689 21548 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.012022 21494 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.012432 21494 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:08 on dist-test-slave-kjk0"
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.015242 21494 fs_report.cc:389] FS layout report
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:39.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/wal
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/data
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:39.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:39.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.093698 21494 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.106930 21494 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:39.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.107496 21494 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.195272 21494 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.254:43435
02:08:39.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.195356 21599 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.254:43435 every 8 connection(s)
02:08:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.198661 21494 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-0/data/info.pb
02:08:39.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.202363 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21494
02:08:39.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:39.203 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.19.202.253:46817
02:08:39.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.209376 20267 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskX8607x/build/tsan/bin/kudu
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46831
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:39.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:08:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.213300 21600 sys_catalog.cc:263] Verifying existing consensus state
02:08:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.221498 21600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap starting.
02:08:39.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.277228 21600 log.cc:826] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Log is configured to *not* fsync() on all Append() calls
02:08:39.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.299669 21600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:39.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.300711 21600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: Bootstrap complete.
02:08:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.329093 21604 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:08:39.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.330071 21604 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.334595 21600 raft_consensus.cc:357] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.338012 21600 raft_consensus.cc:738] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: d47520b5d9b348228a7ae8211f4a16d4, State: Initialized, Role: FOLLOWER
02:08:39.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.339033 21600 consensus_queue.cc:260] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.351305 21604 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:39.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.386200 21607 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:39.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.389319 21600 sys_catalog.cc:564] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: configured and running, proceeding with master startup.
02:08:39.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.390689 21607 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:39 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606115, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:39.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.405733 21439 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:39.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.442786 21575 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b5cff62c0ec44a80abb47b2b4738f998" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4" is_pre_election: true
02:08:39.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.443814 21575 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b5cff62c0ec44a80abb47b2b4738f998 in term 2.
02:08:39.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.445603 21439 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998, d47520b5d9b348228a7ae8211f4a16d4; no voters: 034bbf011b9c4efc93529128399faa68
02:08:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.446465 21604 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Leader pre-election won for term 3
02:08:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.446868 21604 raft_consensus.cc:491] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:08:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.447242 21604 raft_consensus.cc:3058] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 2 FOLLOWER]: Advancing to term 3
02:08:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.459134 21604 raft_consensus.cc:513] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.473770 21575 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b5cff62c0ec44a80abb47b2b4738f998" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "d47520b5d9b348228a7ae8211f4a16d4"
02:08:39.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.474359 21575 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 2 FOLLOWER]: Advancing to term 3
02:08:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.479261 21439 leader_election.cc:336] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111)
02:08:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.480396 21604 leader_election.cc:290] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 election: Requested vote from peers d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435), 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817)
02:08:39.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.494484 21575 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b5cff62c0ec44a80abb47b2b4738f998 in term 3.
02:08:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.495685 21439 leader_election.cc:304] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: b5cff62c0ec44a80abb47b2b4738f998, d47520b5d9b348228a7ae8211f4a16d4; no voters: 034bbf011b9c4efc93529128399faa68
02:08:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.496783 21604 raft_consensus.cc:2802] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 3 FOLLOWER]: Leader election won for term 3
02:08:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.499336 21604 raft_consensus.cc:695] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [term 3 LEADER]: Becoming Leader. State: Replica: b5cff62c0ec44a80abb47b2b4738f998, State: Running, Role: LEADER
02:08:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.500349 21604 consensus_queue.cc:237] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:39.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:39 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606117, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.193@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.529371 21618 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5cff62c0ec44a80abb47b2b4738f998. Latest consensus state: current_term: 3 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.530269 21618 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: LEADER
02:08:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.539489 21624 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
02:08:39.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.545207 20756 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:39.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.552855 21624 catalog_manager.cc:1486] Initializing Kudu cluster ID...
02:08:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.557133 21624 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.557400 21624 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
02:08:39.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.562691 21624 catalog_manager.cc:1506] Loading token signing keys...
02:08:39.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.568548 21624 catalog_manager.cc:5966] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998: Loaded TSK: 0
02:08:39.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.570003 21624 catalog_manager.cc:1516] Initializing in-progress tserver states...
02:08:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.583171 21625 catalog_manager.cc:1261] Loaded cluster ID: 4fae66c10661427685a19797198cb005
02:08:39.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.583469 21625 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: loading cluster ID for follower catalog manager: success
02:08:39.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.591569 21625 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: acquiring CA information for follower catalog manager: success
02:08:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.598572 21625 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:08:39.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:39 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606118, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.254@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.630180 21494 master_runner.cc:186] Error getting master registration for 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111),
02:08:39.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.656447 21494 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.753154 21602 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:08:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.753852 21602 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:08:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.754606 21602 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:08:39.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.803996 21602 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:08:39.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.804330 21602 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:08:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.804821 21602 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:08:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.804972 20898 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.805256 21602 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:08:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.805569 21602 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:08:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.805891 21602 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:08:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.806134 21602 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46081/jwks.json
02:08:39.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.814208 21565 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "5ed97577870d4e989bfaea64821d71d6" instance_seqno: 1756606100410382) as {username='kudu', principal='kudu/127.19.202.194@KRBTEST.COM'} at 127.19.202.194:33981; Asking this server to re-register.
02:08:39.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.816097 20898 heartbeater.cc:461] Registering TS with master...
02:08:39.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.819664 21565 ts_manager.cc:194] Registered new tserver with Master: 5ed97577870d4e989bfaea64821d71d6 (127.19.202.194:33553)
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.868600 21602 master_runner.cc:386] Master server non-default flags:
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.202.212:43147
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.202.254:43435,127.19.202.253:46817,127.19.202.252:36351
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/krb5kdc/kudu.keytab
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.19.202.253
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.202.253:46817
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:08:39.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46081/jwks.json
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.202.253
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46831
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/logs
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:08:39.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
02:08:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:08:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 31 Aug 2025 01:43:21 UTC on 5fd53c4cbb9d
02:08:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7825
02:08:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:08:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.870411 21602 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:08:39.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.871819 21602 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:08:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.891628 21635 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:39.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.926290 21439 consensus_peers.cc:489] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 -> Peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Couldn't send request to peer 034bbf011b9c4efc93529128399faa68. Status: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:08:39.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.964861 21049 heartbeater.cc:344] Connected to a master server at 127.19.202.254:43435
02:08:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.969218 21565 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "077b681b638f472096f34e72869e5de4" instance_seqno: 1756606103450472) as {username='kudu', principal='kudu/127.19.202.195@KRBTEST.COM'} at 127.19.202.195:36501; Asking this server to re-register.
02:08:39.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.970774 21049 heartbeater.cc:461] Registering TS with master...
02:08:39.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:39.974265 21565 ts_manager.cc:194] Registered new tserver with Master: 077b681b638f472096f34e72869e5de4 (127.19.202.195:39229)
02:08:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.109535 21575 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [term 3 FOLLOWER]: Refusing update from remote peer b5cff62c0ec44a80abb47b2b4738f998: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
02:08:40.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.112773 21618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
02:08:40.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:40 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606120, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:40.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.152158 20779 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.177912 21642 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b5cff62c0ec44a80abb47b2b4738f998. Latest consensus state: current_term: 3 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:40.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.179714 21642 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:40.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.223609 21642 sys_catalog.cc:455] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:40.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.224578 21642 sys_catalog.cc:458] T 00000000000000000000000000000000 P d47520b5d9b348228a7ae8211f4a16d4 [sys.catalog]: This master's current role is: FOLLOWER
02:08:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.229856 21618 sys_catalog.cc:455] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "b5cff62c0ec44a80abb47b2b4738f998" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.231148 21618 sys_catalog.cc:458] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 [sys.catalog]: This master's current role is: LEADER
02:08:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.560472 21565 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "72ccd0a1ff544b2a9a0072a776ada85b" instance_seqno: 1756606097930860) as {username='kudu', principal='kudu/127.19.202.193@KRBTEST.COM'} at 127.19.202.193:49157; Asking this server to re-register.
02:08:40.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.565613 20756 heartbeater.cc:461] Registering TS with master...
02:08:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.570482 21565 ts_manager.cc:194] Registered new tserver with Master: 72ccd0a1ff544b2a9a0072a776ada85b (127.19.202.193:46025)
02:08:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.804054 20896 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:40.950992 21047 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:41.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:41.149279 21405 master_runner.cc:186] Error getting master registration for 127.19.202.253:46817: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111),
02:08:41.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:41.149986 21405 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:08:41.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:41.565423 20748 heartbeater.cc:499] Master 127.19.202.252:36351 was elected leader, sending a full tablet report...
02:08:42.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.334594 21439 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111) [suppressed 8 similar messages]
02:08:42.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.341583 21439 consensus_peers.cc:489] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 -> Peer 034bbf011b9c4efc93529128399faa68 (127.19.202.253:46817): Couldn't send request to peer 034bbf011b9c4efc93529128399faa68. Status: Network error: Client connection negotiation failed: client connection to 127.19.202.253:46817: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
02:08:42.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:41.294505 21634 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 21602
02:08:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.704209 21637 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2806 milliseconds
02:08:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.703090 21602 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.806s user 0.000s sys 0.002s
02:08:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.704735 21602 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.808s user 0.000s sys 0.002s
02:08:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:39.895685 21636 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.706205 21602 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:08:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:42.706251 21639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:08:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.710677 21602 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:08:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.713863 21602 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:08:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.715344 21602 hybrid_clock.cc:648] HybridClock initialized: now 1756606122715270 us; error 83 us; skew 500 ppm
02:08:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.716158 21602 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:08:42.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:42 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606122, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:42.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.722541 21602 init.cc:377] Logged in from keytab as kudu/127.19.202.253@KRBTEST.COM (short username kudu)
02:08:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.728492 21602 webserver.cc:480] Webserver started at http://127.19.202.253:46831/ using document root <none> and password file <none>
02:08:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.729595 21602 fs_manager.cc:362] Metadata directory not provided
02:08:42.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.729839 21602 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:08:42.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.738725 21602 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
02:08:42.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.744413 21668 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:08:42.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.746192 21602 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
02:08:42.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.746553 21602 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data,/tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "034bbf011b9c4efc93529128399faa68"
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-08-31 02:08:11 on dist-test-slave-kjk0"
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.748889 21602 fs_report.cc:389] FS layout report
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/wal
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/data
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:08:42.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:08:42.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:08:42.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.820923 21602 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:08:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.827246 21602 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:08:42.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.827752 21602 kserver.cc:163] Server-wide thread pool size limit: 3276
02:08:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.959805 21602 rpc_server.cc:307] RPC server started. Bound to: 127.19.202.253:46817
02:08:42.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.959877 21722 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.202.253:46817 every 8 connection(s)
02:08:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.963356 21602 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskX8607x/test-tmp/mini-kudu-cluster3984624260163848472/master-1/data/info.pb
02:08:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.967983 20267 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskX8607x/build/tsan/bin/kudu as pid 21602
02:08:42.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:08:42.970 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.19.202.252:36351(127.19.202.252:36351)] connecting to peer
02:08:42.974 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.19.202.252:36351(127.19.202.252:36351)] Successfully connected to peer
02:08:42.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.988765 21723 sys_catalog.cc:263] Verifying existing consensus state
02:08:42.997 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.19.202.252
02:08:42.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:42.998479 21723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap starting.
02:08:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:43.020577 21727 negotiation.cc:337] Failed RPC negotiation. Trace:
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.993321 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.19.202.195:37519 (local address 127.19.202.253:46817)
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.005855 (+ 12534us) server_negotiation.cc:206] Beginning negotiation
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.005890 (+ 35us) server_negotiation.cc:399] Waiting for connection header
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.005967 (+ 77us) server_negotiation.cc:407] Connection header received
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.011988 (+ 6021us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.012043 (+ 55us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.012589 (+ 546us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.017185 (+ 4596us) server_negotiation.cc:226] Negotiated authn=SASL
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.018425 (+ 1240us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.19.202.195:37519: BlockingRecv error: recv got EOF from 127.19.202.195:37519 (error 108)
02:08:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11812,"thread_start_us":21676,"threads_started":1}
02:08:43.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.057803 21723 log.cc:826] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Log is configured to *not* fsync() on all Append() calls
02:08:43.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.19.202.252:36351(127.19.202.252:36351)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
02:08:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:43.077975 21725 negotiation.cc:337] Failed RPC negotiation. Trace:
02:08:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.974302 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46972 (local address 127.19.202.252:36351)
02:08:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.981758 (+ 7456us) server_negotiation.cc:206] Beginning negotiation
02:08:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.981790 (+ 32us) server_negotiation.cc:399] Waiting for connection header
02:08:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.981844 (+ 54us) server_negotiation.cc:407] Connection header received
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.981989 (+ 145us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.982020 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.989262 (+ 7242us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:42.989541 (+ 279us) server_negotiation.cc:226] Negotiated authn=SASL
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.013078 (+ 23537us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.014786 (+ 1708us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.064775 (+ 49989us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.065428 (+ 653us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0831 02:08:43.076254 (+ 10826us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46972: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:46972): Cannot send after transport endpoint shutdown (error 108)
02:08:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":6827,"thread_start_us":7798,"threads_started":1}
02:08:43.080 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.202.253:46817(127.19.202.253:46817)] disconnecting while in state TERMINATED
02:08:43.082 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.19.202.252:36351(127.19.202.252:36351)] handling channelInactive
02:08:43.083 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.202.254:43435(127.19.202.254:43435)] disconnecting while in state TERMINATED
02:08:43.083 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.19.202.252:36351(127.19.202.252:36351)] disconnecting while in state TERMINATED
02:08:43.087 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:08:43.089 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:08:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.097580 21723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:08:43.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.098419 21723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68: Bootstrap complete.
02:08:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.109690 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20614
02:08:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:43 dist-test-slave-kjk0 krb5kdc[20279](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606123, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:08:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.132864 20930 init.cc:295] Successfully reacquired a new kerberos TGT
02:08:43.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.131322 21723 raft_consensus.cc:357] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.134825 21723 raft_consensus.cc:738] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 034bbf011b9c4efc93529128399faa68, State: Initialized, Role: FOLLOWER
02:08:43.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.136062 21723 consensus_queue.cc:260] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } }
02:08:43.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.150456 21732 sys_catalog.cc:455] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d47520b5d9b348228a7ae8211f4a16d4" member_type: VOTER last_known_addr { host: "127.19.202.254" port: 43435 } } peers { permanent_uuid: "034bbf011b9c4efc93529128399faa68" member_type: VOTER last_known_addr { host: "127.19.202.253" port: 46817 } } peers { permanent_uuid: "b5cff62c0ec44a80abb47b2b4738f998" member_type: VOTER last_known_addr { host: "127.19.202.252" port: 36351 } } }
02:08:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.153059 21732 sys_catalog.cc:458] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: This master's current role is: FOLLOWER
02:08:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.152626 21723 sys_catalog.cc:564] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [sys.catalog]: configured and running, proceeding with master startup.
02:08:43.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.155356 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20763
02:08:43.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.219334 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20911
02:08:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.228521 21743 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:08:43.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.297281 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21494
02:08:43.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:43 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606122, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.253@KRBTEST.COM for kudu/127.19.202.254@KRBTEST.COM
02:08:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 31 02:08:43 dist-test-slave-kjk0 krb5kdc[20279](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756606115, etypes {rep=17 tkt=17 ses=17}, kudu/127.19.202.252@KRBTEST.COM for kudu/127.19.202.253@KRBTEST.COM
02:08:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.333153 21698 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 034bbf011b9c4efc93529128399faa68 [term 2 FOLLOWER]: Advancing to term 3
02:08:43.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:43.336691 21439 connection.cc:537] client connection to 127.19.202.254:43435 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:08:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.338531 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21602
02:08:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250831 02:08:43.340253 21439 consensus_peers.cc:489] T 00000000000000000000000000000000 P b5cff62c0ec44a80abb47b2b4738f998 -> Peer d47520b5d9b348228a7ae8211f4a16d4 (127.19.202.254:43435): Couldn't send request to peer d47520b5d9b348228a7ae8211f4a16d4. Status: Network error: Client connection negotiation failed: client connection to 127.19.202.254:43435: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:08:43.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:43.360944 20267 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 21405
02:08:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:08:44Z chronyd exiting
02:08:45.628 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:08:45.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.629891 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 19844
02:08:45.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.657907 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 19979
02:08:45.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.685606 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 20126
02:08:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.713759 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 19557
02:08:45.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.745577 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 19627
02:08:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250831 02:08:45.777855 19545 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskX8607x/build/tsan/bin/kudu with pid 19702
02:08:46.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-08-31T02:08:46Z chronyd exiting
02:08:47.961 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:975) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) ~[?:1.8.0_275]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[?:1.8.0_275]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:975)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162)
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189)
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
... 52 more
02:08:49.259 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:08:49.260 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:975) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) ~[?:1.8.0_275]
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189) ~[?:1.8.0_275]
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) ~[?:1.8.0_275]
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) ~[?:1.8.0_275]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192) ~[?:1.8.0_275]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959) ~[main/:?]
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956) ~[main/:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_275]
at javax.security.auth.Subject.doAs(Subject.java:422) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94) ~[main/:?]
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113) ~[main/:?]
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955) ~[main/:?]
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845) ~[main/:?]
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:975)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:211)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:959)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:956)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:955)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:845)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:782)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:731)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:379)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:162)
at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122)
at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:189)
at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212)
at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179)
at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:192)
... 52 more