15:06:28.375 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:06:28.387 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrL_IJG/test-tmp
15:06:28.390 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:06:28.394 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrL_IJG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:06:28.719 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:06:28.841 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:06:28.842 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:06:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:28.846117 17544 mini_oidc.cc:150] Starting JWKS server
15:06:28.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:28.849658 17544 webserver.cc:492] Webserver started at http://127.0.0.1:42491/ using document root <none> and password file <none>
15:06:28.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:28.850176 17544 mini_oidc.cc:186] Starting OIDC Discovery server
15:06:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:28.852057 17544 webserver.cc:492] Webserver started at http://127.0.0.1:37857/ using document root <none> and password file <none>
15:06:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:06:28Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:06:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:06:28Z Disabled control of system clock
15:06:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:28.912789 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal
15:06:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data
15:06:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:28.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/logs
15:06:28.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data/info.pb
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:28.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:28.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.62:44789
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.62
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:28.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:28.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json with env {}
15:06:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.207087 17556 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.207625 17556 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.208195 17556 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:29.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.239284 17556 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.239620 17556 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.239862 17556 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.240144 17556 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.240337 17556 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.240474 17556 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42491/jwks.json
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:29.275645 17556 master_runner.cc:386] Master server non-default flags:
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:29.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data
15:06:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal
15:06:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.62:44789
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json
15:06:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data/info.pb
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.62
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/logs
15:06:29.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:29.277100 17556 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:29.279239 17556 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:29.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.289969 17562 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:29.291476 17563 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.480432 17565 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:30.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.483341 17564 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1187 milliseconds
15:06:30.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.483438 17556 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:30.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.484820 17556 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:30.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.487491 17556 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:30.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.488863 17556 hybrid_clock.cc:648] HybridClock initialized: now 1765638390488839 us; error 41 us; skew 500 ppm
15:06:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.489773 17556 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.496146 17556 webserver.cc:492] Webserver started at http://127.17.34.62:36605/ using document root <none> and password file <none>
15:06:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.497196 17556 fs_manager.cc:362] Metadata directory not provided
15:06:30.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.497431 17556 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.497967 17556 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:30.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.502272 17556 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data/instance:
15:06:30.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef623ef80f8642d4bfa22646dcc04ca3"
15:06:30.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:30 on dist-test-slave-xht5"
15:06:30.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.503643 17556 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal/instance:
15:06:30.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef623ef80f8642d4bfa22646dcc04ca3"
15:06:30.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:30 on dist-test-slave-xht5"
15:06:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.513245 17556 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
15:06:30.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.519373 17572 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:30.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.520628 17556 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:06:30.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.521030 17556 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal
15:06:30.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef623ef80f8642d4bfa22646dcc04ca3"
15:06:30.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:30 on dist-test-slave-xht5"
15:06:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.521456 17556 fs_report.cc:389] FS layout report
15:06:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal
15:06:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal
15:06:30.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data/data
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.569736 17556 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:30.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.571370 17556 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.571892 17556 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.643239 17556 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.62:44789
15:06:30.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.643301 17623 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.62:44789 every 8 connection(s)
15:06:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.646106 17556 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/data/info.pb
15:06:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.651986 17624 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.652433 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 17556
15:06:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.652812 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-0/wal/instance
15:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.656104 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal
15:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data
15:06:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/logs
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data/info.pb
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:30.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:30.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.61:42271
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.61
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json with env {}
15:06:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.665019 17624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } has no permanent_uuid. Determining permanent_uuid...
15:06:30.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.689429 17624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } has no permanent_uuid. Determining permanent_uuid...
15:06:30.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.691972 17576 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:30.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.696312 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.745942 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 1
15:06:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.750000 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:30.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.818517 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 2
15:06:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.823154 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:30.914527 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 3
15:06:30.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.918514 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:30.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.964679 17626 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:30.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.965226 17626 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:30.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.965828 17626 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.997792 17626 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.998091 17626 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.998286 17626 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.998553 17626 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.998787 17626 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:30.998998 17626 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42491/jwks.json
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.036458 17626 master_runner.cc:386] Master server non-default flags:
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.61:42271
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:31.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data/info.pb
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.61
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/logs
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.037753 17626 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:31.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.039940 17626 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:31.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:31.051786 17635 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.061899 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 4
15:06:31.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:31.066435 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:31.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.365886 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 5
15:06:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:31.371528 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:31.918946 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 6
15:06:31.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:31.926867 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.61:42271: Network error: Client connection negotiation failed: client connection to 127.17.34.61:42271: connect: Connection refused (error 111)
15:06:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:31.053566 17636 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:32.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.245438 17637 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:06:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.246954 17638 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.247532 17626 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:32.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.250355 17626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.253630 17626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.255045 17626 hybrid_clock.cc:648] HybridClock initialized: now 1765638392255003 us; error 54 us; skew 500 ppm
15:06:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.256042 17626 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.262576 17626 webserver.cc:492] Webserver started at http://127.17.34.61:41217/ using document root <none> and password file <none>
15:06:32.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.263480 17626 fs_manager.cc:362] Metadata directory not provided
15:06:32.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.263669 17626 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.264038 17626 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.268304 17626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data/instance:
15:06:32.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d346dcd2795248c580d6d19f0d959105"
15:06:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:32 on dist-test-slave-xht5"
15:06:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.269598 17626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal/instance:
15:06:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d346dcd2795248c580d6d19f0d959105"
15:06:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:32 on dist-test-slave-xht5"
15:06:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.276978 17626 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:06:32.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.282629 17647 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.283725 17626 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:06:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.284072 17626 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal
15:06:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d346dcd2795248c580d6d19f0d959105"
15:06:32.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:32 on dist-test-slave-xht5"
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.284453 17626 fs_report.cc:389] FS layout report
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data/data
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:32.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.342100 17626 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:32.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.343832 17626 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:32.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.344331 17626 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:32.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.414495 17626 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.61:42271
15:06:32.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.414588 17698 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.61:42271 every 8 connection(s)
15:06:32.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.417133 17626 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/data/info.pb
15:06:32.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.422941 17699 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.425271 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 17626
15:06:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.425706 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-1/wal/instance
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.427765 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/logs
15:06:32.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data/info.pb
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.60:36689
15:06:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.60
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json with env {}
15:06:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.433992 17699 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } has no permanent_uuid. Determining permanent_uuid...
15:06:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.461819 17699 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } has no permanent_uuid. Determining permanent_uuid...
15:06:32.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.477922 17699 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } has no permanent_uuid. Determining permanent_uuid...
15:06:32.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.480288 17648 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.484726 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.534221 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 1
15:06:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.538194 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.606627 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 2
15:06:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.610694 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.702073 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 3
15:06:32.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.707235 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.752094 17701 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.752652 17701 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:32.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.753113 17701 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.785305 17701 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:32.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.785610 17701 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.785801 17701 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.785982 17701 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.786161 17701 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.786312 17701 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42491/jwks.json
15:06:32.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.822379 17701 master_runner.cc:386] Master server non-default flags:
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.60:36689
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:32.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42491/jwks.json
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data/info.pb
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.60
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/logs
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:32.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:32.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.823951 17701 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:32.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.825982 17701 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:32.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.837015 17711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:32.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.850636 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 4
15:06:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.856904 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:32.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:32.987267 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } attempt: 7
15:06:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.017939 17624 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } has no permanent_uuid. Determining permanent_uuid...
15:06:33.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.027421 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.086109 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 1
15:06:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.093148 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.156437 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 5
15:06:33.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.163064 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.174767 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 2
15:06:33.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.181329 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.266916 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 3
15:06:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.273577 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.414183 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 4
15:06:33.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.423311 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.706940 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 5
15:06:33.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:33.710719 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 6
15:06:33.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.717041 17624 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:33.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:33.724143 17699 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.34.60:36689: Network error: Client connection negotiation failed: client connection to 127.17.34.60:36689: connect: Connection refused (error 111)
15:06:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:32.838925 17712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.061627 17714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.064431 17713 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1221 milliseconds
15:06:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.064551 17701 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.065971 17701 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.069092 17701 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:34.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.070541 17701 hybrid_clock.cc:648] HybridClock initialized: now 1765638394070497 us; error 62 us; skew 500 ppm
15:06:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.071492 17701 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.079563 17701 webserver.cc:492] Webserver started at http://127.17.34.60:44193/ using document root <none> and password file <none>
15:06:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.080834 17701 fs_manager.cc:362] Metadata directory not provided
15:06:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.081141 17701 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:34.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.081681 17701 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.086107 17701 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data/instance:
15:06:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfc630e1a3124e3cad2ee2be22c123be"
15:06:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:34 on dist-test-slave-xht5"
15:06:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.087420 17701 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal/instance:
15:06:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfc630e1a3124e3cad2ee2be22c123be"
15:06:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:34 on dist-test-slave-xht5"
15:06:34.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.094866 17701 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:06:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.100814 17726 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.101999 17701 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:06:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.102316 17701 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal
15:06:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfc630e1a3124e3cad2ee2be22c123be"
15:06:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:34 on dist-test-slave-xht5"
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.102682 17701 fs_report.cc:389] FS layout report
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data/data
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:34.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:34.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.156265 17701 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.157812 17701 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:34.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.158231 17701 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.228931 17701 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.60:36689
15:06:34.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.228976 17777 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.60:36689 every 8 connection(s)
15:06:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.232452 17701 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/data/info.pb
15:06:34.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.238323 17778 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:34.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.239432 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 17701
15:06:34.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.239840 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/master-2/wal/instance
15:06:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.253810 17778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } has no permanent_uuid. Determining permanent_uuid...
15:06:34.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.269661 17624 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 6
15:06:34.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.284528 17778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } has no permanent_uuid. Determining permanent_uuid...
15:06:34.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.298161 17778 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } has no permanent_uuid. Determining permanent_uuid...
15:06:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.303684 17624 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: Bootstrap starting.
15:06:34.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.313201 17624 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: Neither blocks nor log segments found. Creating new log.
15:06:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.315790 17624 log.cc:826] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: Log is configured to *not* fsync() on all Append() calls
15:06:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.318284 17778 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: Bootstrap starting.
15:06:34.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.323817 17624 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: No bootstrap required, opened a new log
15:06:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.326634 17778 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: Neither blocks nor log segments found. Creating new log.
15:06:34.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.329082 17778 log.cc:826] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: Log is configured to *not* fsync() on all Append() calls
15:06:34.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.336999 17778 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: No bootstrap required, opened a new log
15:06:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.358000 17624 raft_consensus.cc:359] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.358737 17624 raft_consensus.cc:385] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.359027 17624 raft_consensus.cc:740] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef623ef80f8642d4bfa22646dcc04ca3, State: Initialized, Role: FOLLOWER
15:06:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.359952 17624 consensus_queue.cc:260] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.361956 17778 raft_consensus.cc:359] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.363250 17778 raft_consensus.cc:385] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:34.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.362767 17790 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.363620 17790 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: This master's current role is: FOLLOWER
15:06:34.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.363581 17778 raft_consensus.cc:740] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfc630e1a3124e3cad2ee2be22c123be, State: Initialized, Role: FOLLOWER
15:06:34.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.364689 17624 sys_catalog.cc:565] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: configured and running, proceeding with master startup.
15:06:34.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.364607 17778 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.368222 17791 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.369907 17791 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: This master's current role is: FOLLOWER
15:06:34.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.370752 17778 sys_catalog.cc:565] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: configured and running, proceeding with master startup.
15:06:34.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.406154 17812 catalog_manager.cc:1568] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.406903 17812 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.411365 17813 catalog_manager.cc:1568] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.411772 17813 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.413213 17732 tablet.cc:2378] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:34.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.404404 17556 master_runner.cc:186] Error getting master registration for 127.17.34.61:42271: OK, instance_id { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" instance_seqno: 1765638392395528 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.465904 17556 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.414741 17701 master_runner.cc:186] Error getting master registration for 127.17.34.61:42271: OK, instance_id { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" instance_seqno: 1765638392395528 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.472049 17701 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:34.581559 17578 tablet.cc:2378] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.611019 17790 raft_consensus.cc:493] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:34.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.611516 17790 raft_consensus.cc:515] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.613817 17790 leader_election.cc:290] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d346dcd2795248c580d6d19f0d959105 (127.17.34.61:42271), dfc630e1a3124e3cad2ee2be22c123be (127.17.34.60:36689)
15:06:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.615820 17753 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dfc630e1a3124e3cad2ee2be22c123be" is_pre_election: true
15:06:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.616734 17753 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef623ef80f8642d4bfa22646dcc04ca3 in term 0.
15:06:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.618268 17573 leader_election.cc:304] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: dfc630e1a3124e3cad2ee2be22c123be, ef623ef80f8642d4bfa22646dcc04ca3; no voters:
15:06:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.619722 17790 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.620261 17790 raft_consensus.cc:493] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.620609 17790 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 0 FOLLOWER]: Advancing to term 1
15:06:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.631420 17790 raft_consensus.cc:515] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.633788 17790 leader_election.cc:290] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [CANDIDATE]: Term 1 election: Requested vote from peers d346dcd2795248c580d6d19f0d959105 (127.17.34.61:42271), dfc630e1a3124e3cad2ee2be22c123be (127.17.34.60:36689)
15:06:34.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.635177 17753 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dfc630e1a3124e3cad2ee2be22c123be"
15:06:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.635761 17753 raft_consensus.cc:3060] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 0 FOLLOWER]: Advancing to term 1
15:06:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.644393 17753 raft_consensus.cc:2468] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef623ef80f8642d4bfa22646dcc04ca3 in term 1.
15:06:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.646245 17573 leader_election.cc:304] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: dfc630e1a3124e3cad2ee2be22c123be, ef623ef80f8642d4bfa22646dcc04ca3; no voters:
15:06:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.647701 17790 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 1 FOLLOWER]: Leader election won for term 1
15:06:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.658087 17790 raft_consensus.cc:697] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [term 1 LEADER]: Becoming Leader. State: Replica: ef623ef80f8642d4bfa22646dcc04ca3, State: Running, Role: LEADER
15:06:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.659561 17790 consensus_queue.cc:237] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.680922 17816 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef623ef80f8642d4bfa22646dcc04ca3. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.681787 17816 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: This master's current role is: LEADER
15:06:34.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.700049 17818 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:06:34.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.715600 17818 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:06:34.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.738005 17753 raft_consensus.cc:1275] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [term 1 FOLLOWER]: Refusing update from remote peer ef623ef80f8642d4bfa22646dcc04ca3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:34.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.739972 17790 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:34.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.761747 17791 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef623ef80f8642d4bfa22646dcc04ca3. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.764850 17791 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: This master's current role is: FOLLOWER
15:06:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.774051 17819 mvcc.cc:204] Tried to move back new op lower bound from 7232054864823795712 to 7232054864537899008. Current Snapshot: MvccSnapshot[applied={T|T < 7232054864823795712}]
15:06:34.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.782939 17816 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.783690 17816 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: This master's current role is: LEADER
15:06:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.783957 17791 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.784691 17791 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be [sys.catalog]: This master's current role is: FOLLOWER
15:06:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.784641 17699 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } attempt: 7
15:06:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.789376 17818 catalog_manager.cc:1357] Generated new cluster ID: f7d196d523eb427db21fca7d6d2e9eba
15:06:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.789702 17818 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:06:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.784334 17820 mvcc.cc:204] Tried to move back new op lower bound from 7232054864823795712 to 7232054864537899008. Current Snapshot: MvccSnapshot[applied={T|T < 7232054864823795712}]
15:06:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.853798 17699 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: Bootstrap starting.
15:06:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.858566 17818 catalog_manager.cc:1380] Generated new certificate authority record
15:06:34.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.860920 17818 catalog_manager.cc:1514] Loading token signing keys...
15:06:34.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.873572 17699 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: Neither blocks nor log segments found. Creating new log.
15:06:34.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.881989 17699 log.cc:826] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: Log is configured to *not* fsync() on all Append() calls
15:06:34.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.899513 17699 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: No bootstrap required, opened a new log
15:06:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.903504 17818 catalog_manager.cc:6027] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3: Generated new TSK 0
15:06:34.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.904645 17818 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:06:34.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.936659 17699 raft_consensus.cc:359] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.937570 17699 raft_consensus.cc:385] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.937896 17699 raft_consensus.cc:740] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d346dcd2795248c580d6d19f0d959105, State: Initialized, Role: FOLLOWER
15:06:34.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.938943 17699 consensus_queue.cc:260] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } }
15:06:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.948666 17699 sys_catalog.cc:565] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: configured and running, proceeding with master startup.
15:06:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.949676 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/logs
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data/info.pb
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.1:0
15:06:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.1
15:06:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.1
15:06:34.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:06:34.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.953774 17674 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d346dcd2795248c580d6d19f0d959105" is_pre_election: true
15:06:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.956112 17672 raft_consensus.cc:3060] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [term 0 FOLLOWER]: Advancing to term 1
15:06:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.954463 17673 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d346dcd2795248c580d6d19f0d959105"
15:06:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.945830 17826 sys_catalog.cc:455] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:34.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.958221 17826 sys_catalog.cc:458] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: This master's current role is: FOLLOWER
15:06:34.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.967119 17672 raft_consensus.cc:1275] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [term 1 FOLLOWER]: Refusing update from remote peer ef623ef80f8642d4bfa22646dcc04ca3: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:34.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:34.972344 17816 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.068989 17826 sys_catalog.cc:455] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef623ef80f8642d4bfa22646dcc04ca3. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.069128 17835 sys_catalog.cc:455] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.069789 17826 sys_catalog.cc:458] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: This master's current role is: FOLLOWER
15:06:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.069862 17835 sys_catalog.cc:458] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105 [sys.catalog]: This master's current role is: FOLLOWER
15:06:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.093533 17843 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:06:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.417665 17813 catalog_manager.cc:1269] Loaded cluster ID: f7d196d523eb427db21fca7d6d2e9eba
15:06:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.418439 17813 catalog_manager.cc:1562] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: loading cluster ID for follower catalog manager: success
15:06:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.426192 17813 catalog_manager.cc:1584] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: acquiring CA information for follower catalog manager: success
15:06:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.432124 17813 catalog_manager.cc:1612] T 00000000000000000000000000000000 P dfc630e1a3124e3cad2ee2be22c123be: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.518798 17828 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:35.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.519605 17828 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.520527 17828 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.574566 17816 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef623ef80f8642d4bfa22646dcc04ca3" member_type: VOTER last_known_addr { host: "127.17.34.62" port: 44789 } } peers { permanent_uuid: "d346dcd2795248c580d6d19f0d959105" member_type: VOTER last_known_addr { host: "127.17.34.61" port: 42271 } } peers { permanent_uuid: "dfc630e1a3124e3cad2ee2be22c123be" member_type: VOTER last_known_addr { host: "127.17.34.60" port: 36689 } } }
15:06:35.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.576189 17816 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef623ef80f8642d4bfa22646dcc04ca3 [sys.catalog]: This master's current role is: LEADER
15:06:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.579428 17828 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.580804 17828 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.34.1
15:06:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.645437 17828 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.1:0
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data/info.pb
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.1
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.1
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/logs
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.647127 17828 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:35.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:35.649161 17828 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.687340 17851 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:36.113127 17843 catalog_manager.cc:1269] Loaded cluster ID: f7d196d523eb427db21fca7d6d2e9eba
15:06:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:36.113437 17843 catalog_manager.cc:1562] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: loading cluster ID for follower catalog manager: success
15:06:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:36.133530 17843 catalog_manager.cc:1584] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: acquiring CA information for follower catalog manager: success
15:06:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:36.139714 17843 catalog_manager.cc:1612] T 00000000000000000000000000000000 P d346dcd2795248c580d6d19f0d959105: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:37.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:35.690685 17852 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:37.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:37.077090 17850 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 17828
15:06:37.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:37.746165 17828 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.058s user 0.630s sys 1.184s
15:06:37.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:37.750562 17828 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.062s user 0.631s sys 1.184s
15:06:37.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.755666 17828 server_base.cc:1047] running on GCE node
15:06:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:37.757334 17857 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:37.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.758826 17828 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.772665 17828 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.774120 17828 hybrid_clock.cc:648] HybridClock initialized: now 1765638397773886 us; error 249 us; skew 500 ppm
15:06:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.775513 17828 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:37.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.807971 17828 webserver.cc:492] Webserver started at http://127.17.34.1:36661/ using document root <none> and password file <none>
15:06:37.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.809602 17828 fs_manager.cc:362] Metadata directory not provided
15:06:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.810101 17828 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:37.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.810820 17828 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:37.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.829456 17828 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data/instance:
15:06:37.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e144abb402cf4d219180c3d8714cb9e2"
15:06:37.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:37 on dist-test-slave-xht5"
15:06:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.839313 17828 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal/instance:
15:06:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e144abb402cf4d219180c3d8714cb9e2"
15:06:37.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:37 on dist-test-slave-xht5"
15:06:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.857427 17828 fs_manager.cc:696] Time spent creating directory manager: real 0.017s user 0.009s sys 0.005s
15:06:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.873334 17863 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:37.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.875262 17828 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.002s sys 0.007s
15:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.875921 17828 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal
15:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e144abb402cf4d219180c3d8714cb9e2"
15:06:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:37 on dist-test-slave-xht5"
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.884414 17828 fs_report.cc:389] FS layout report
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data/data
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:37.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:37.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:37.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.956035 17828 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.958102 17828 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.958688 17828 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.962075 17828 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.968031 17828 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.968327 17828 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.968613 17828 ts_tablet_manager.cc:616] Registered 0 tablets
15:06:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:37.968813 17828 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.164633 17828 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.1:37363
15:06:38.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.164703 17975 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.1:37363 every 8 connection(s)
15:06:38.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.168193 17828 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/data/info.pb
15:06:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.171221 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 17828
15:06:38.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.171748 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-0/wal/instance
15:06:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.202857 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal
15:06:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:38.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/logs
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data/info.pb
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.2:0
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.2
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.2
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:06:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.225016 17978 heartbeater.cc:344] Connected to a master server at 127.17.34.62:44789
15:06:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.225549 17978 heartbeater.cc:461] Registering TS with master...
15:06:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.226608 17976 heartbeater.cc:344] Connected to a master server at 127.17.34.60:36689
15:06:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.226926 17978 heartbeater.cc:507] Master 127.17.34.62:44789 requested a full tablet report, sending...
15:06:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.226959 17976 heartbeater.cc:461] Registering TS with master...
15:06:38.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.229589 17976 heartbeater.cc:507] Master 127.17.34.60:36689 requested a full tablet report, sending...
15:06:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.230635 17977 heartbeater.cc:344] Connected to a master server at 127.17.34.61:42271
15:06:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.231055 17977 heartbeater.cc:461] Registering TS with master...
15:06:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.232132 17977 heartbeater.cc:507] Master 127.17.34.61:42271 requested a full tablet report, sending...
15:06:38.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.232820 17589 ts_manager.cc:194] Registered new tserver with Master: e144abb402cf4d219180c3d8714cb9e2 (127.17.34.1:37363)
15:06:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.235460 17664 ts_manager.cc:194] Registered new tserver with Master: e144abb402cf4d219180c3d8714cb9e2 (127.17.34.1:37363)
15:06:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.237401 17743 ts_manager.cc:194] Registered new tserver with Master: e144abb402cf4d219180c3d8714cb9e2 (127.17.34.1:37363)
15:06:38.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.241065 17589 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.17.34.1:60881
15:06:38.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.569470 17986 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.569941 17986 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.570376 17986 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.602619 17986 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.603572 17986 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.34.2
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.639873 17986 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.2:0
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:38.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data/info.pb
15:06:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.2
15:06:38.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:38.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.2
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/logs
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.641201 17986 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:38.643427 17986 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:38.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.664490 17992 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:39.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:39.245657 17978 heartbeater.cc:499] Master 127.17.34.62:44789 was elected leader, sending a full tablet report...
15:06:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:38.667829 17993 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:40.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.067350 17991 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 17986
15:06:40.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.090489 17986 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.424s user 0.563s sys 0.838s
15:06:40.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.091142 17986 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.425s user 0.563s sys 0.838s
15:06:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.099125 17999 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.099123 17986 server_base.cc:1047] running on GCE node
15:06:40.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.100654 17986 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:40.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.102732 17986 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:40.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.104111 17986 hybrid_clock.cc:648] HybridClock initialized: now 1765638400104071 us; error 32 us; skew 500 ppm
15:06:40.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.105041 17986 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:40.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.110735 17986 webserver.cc:492] Webserver started at http://127.17.34.2:42087/ using document root <none> and password file <none>
15:06:40.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.111642 17986 fs_manager.cc:362] Metadata directory not provided
15:06:40.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.111837 17986 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:40.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.112296 17986 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:40.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.116457 17986 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data/instance:
15:06:40.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18593f79a0b146528e276da32dfb78e0"
15:06:40.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:40 on dist-test-slave-xht5"
15:06:40.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.117498 17986 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal/instance:
15:06:40.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18593f79a0b146528e276da32dfb78e0"
15:06:40.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:40 on dist-test-slave-xht5"
15:06:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.124423 17986 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
15:06:40.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.129810 18004 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.130842 17986 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
15:06:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.131155 17986 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal
15:06:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "18593f79a0b146528e276da32dfb78e0"
15:06:40.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:40 on dist-test-slave-xht5"
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.131470 17986 fs_report.cc:389] FS layout report
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data/data
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:40.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.191767 17986 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:40.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.193259 17986 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:40.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.193744 17986 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:40.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.196282 17986 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:40.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.200416 17986 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:40.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.200623 17986 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.200923 17986 ts_tablet_manager.cc:616] Registered 0 tablets
15:06:40.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.201130 17986 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.346210 17986 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.2:35427
15:06:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.346313 18116 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.2:35427 every 8 connection(s)
15:06:40.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.348755 17986 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/data/info.pb
15:06:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.361089 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 17986
15:06:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.361937 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-1/wal/instance
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.375169 17544 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/logs
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data/info.pb
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.3:0
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.3
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.3
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:40.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:06:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.391753 18117 heartbeater.cc:344] Connected to a master server at 127.17.34.60:36689
15:06:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.392256 18117 heartbeater.cc:461] Registering TS with master...
15:06:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.393213 18118 heartbeater.cc:344] Connected to a master server at 127.17.34.61:42271
15:06:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.393546 18118 heartbeater.cc:461] Registering TS with master...
15:06:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.393801 18117 heartbeater.cc:507] Master 127.17.34.60:36689 requested a full tablet report, sending...
15:06:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.394439 18118 heartbeater.cc:507] Master 127.17.34.61:42271 requested a full tablet report, sending...
15:06:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.396751 17664 ts_manager.cc:194] Registered new tserver with Master: 18593f79a0b146528e276da32dfb78e0 (127.17.34.2:35427)
15:06:40.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.397768 17743 ts_manager.cc:194] Registered new tserver with Master: 18593f79a0b146528e276da32dfb78e0 (127.17.34.2:35427)
15:06:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.399134 18119 heartbeater.cc:344] Connected to a master server at 127.17.34.62:44789
15:06:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.399592 18119 heartbeater.cc:461] Registering TS with master...
15:06:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.400322 18119 heartbeater.cc:507] Master 127.17.34.62:44789 requested a full tablet report, sending...
15:06:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.402199 17589 ts_manager.cc:194] Registered new tserver with Master: 18593f79a0b146528e276da32dfb78e0 (127.17.34.2:35427)
15:06:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.403525 17589 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.17.34.2:57261
15:06:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.710095 18127 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.710650 18127 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.711217 18127 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.743641 18127 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:40.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.744604 18127 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.34.3
15:06:40.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.781523 18127 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.34.20:39785
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.34.3:0
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data/info.pb
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.34.3
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.34.62:44789,127.17.34.61:42271,127.17.34.60:36689
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.34.3
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/logs
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.783205 18127 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:40.784825 18127 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:40.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.796643 18133 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:41.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:41.406386 18119 heartbeater.cc:499] Master 127.17.34.62:44789 was elected leader, sending a full tablet report...
15:06:42.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:40.797204 18134 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:42.087062 18136 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:42.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:42.091008 18135 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1293 milliseconds
15:06:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:42.091616 18127 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.295s user 0.408s sys 0.878s
15:06:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:42.091852 18127 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.295s user 0.408s sys 0.878s
15:06:42.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.092017 18127 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:42.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.093086 18127 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.095562 18127 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.096909 18127 hybrid_clock.cc:648] HybridClock initialized: now 1765638402096877 us; error 50 us; skew 500 ppm
15:06:42.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.097730 18127 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.105151 18127 webserver.cc:492] Webserver started at http://127.17.34.3:36127/ using document root <none> and password file <none>
15:06:42.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.106231 18127 fs_manager.cc:362] Metadata directory not provided
15:06:42.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.106511 18127 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:42.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.106999 18127 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.111565 18127 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data/instance:
15:06:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae5b4fb8eb14a6b8b524b0359002934"
15:06:42.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:42 on dist-test-slave-xht5"
15:06:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.112739 18127 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal/instance:
15:06:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae5b4fb8eb14a6b8b524b0359002934"
15:06:42.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:42 on dist-test-slave-xht5"
15:06:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.120867 18127 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
15:06:42.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.126830 18144 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.128115 18127 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
15:06:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.128476 18127 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal
15:06:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ae5b4fb8eb14a6b8b524b0359002934"
15:06:42.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:42 on dist-test-slave-xht5"
15:06:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.128819 18127 fs_report.cc:389] FS layout report
15:06:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal
15:06:42.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data/data
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:42.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:42.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.217485 18127 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.219045 18127 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:42.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.219555 18127 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:42.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.222041 18127 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.227005 18127 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.227279 18127 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.227571 18127 ts_tablet_manager.cc:616] Registered 0 tablets
15:06:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.227780 18127 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.380977 18127 rpc_server.cc:307] RPC server started. Bound to: 127.17.34.3:36079
15:06:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.381083 18256 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.34.3:36079 every 8 connection(s)
15:06:42.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.383488 18127 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/data/info.pb
15:06:42.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.386860 17544 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18127
15:06:42.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.387377 17544 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster8734775218068327184/ts-2/wal/instance
15:06:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.433049 18257 heartbeater.cc:344] Connected to a master server at 127.17.34.60:36689
15:06:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.433605 18257 heartbeater.cc:461] Registering TS with master...
15:06:42.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.434990 18257 heartbeater.cc:507] Master 127.17.34.60:36689 requested a full tablet report, sending...
15:06:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.437315 18259 heartbeater.cc:344] Connected to a master server at 127.17.34.62:44789
15:06:42.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.437657 18259 heartbeater.cc:461] Registering TS with master...
15:06:42.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.438277 17743 ts_manager.cc:194] Registered new tserver with Master: 5ae5b4fb8eb14a6b8b524b0359002934 (127.17.34.3:36079)
15:06:42.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.438596 18259 heartbeater.cc:507] Master 127.17.34.62:44789 requested a full tablet report, sending...
15:06:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.441357 18258 heartbeater.cc:344] Connected to a master server at 127.17.34.61:42271
15:06:42.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.441699 18258 heartbeater.cc:461] Registering TS with master...
15:06:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.441931 17589 ts_manager.cc:194] Registered new tserver with Master: 5ae5b4fb8eb14a6b8b524b0359002934 (127.17.34.3:36079)
15:06:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.442617 18258 heartbeater.cc:507] Master 127.17.34.61:42271 requested a full tablet report, sending...
15:06:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.443313 17589 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.17.34.3:48905
15:06:42.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.445326 17664 ts_manager.cc:194] Registered new tserver with Master: 5ae5b4fb8eb14a6b8b524b0359002934 (127.17.34.3:36079)
15:06:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:42.456936 17544 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:06:42.469 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:06:42.472 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:06:42.482 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.34.62"
port: 44789
}
bound_http_address {
host: "127.17.34.62"
port: 36605
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.34.61"
port: 42271
}
bound_http_address {
host: "127.17.34.61"
port: 41217
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.34.60"
port: 36689
}
bound_http_address {
host: "127.17.34.60"
port: 44193
}
}
}
15:06:42.486 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:06:42.489 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.34.1"
port: 37363
}
bound_http_address {
host: "127.17.34.1"
port: 36661
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.34.2"
port: 35427
}
bound_http_address {
host: "127.17.34.2"
port: 42087
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.34.3"
port: 36079
}
bound_http_address {
host: "127.17.34.3"
port: 36127
}
}
}
15:06:42.489 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:06:42.656 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
15:06:42.681 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:06:42.809 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrL_IJG/test-tmp
15:06:42.810 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:06:42.810 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrL_IJG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:06:42.812 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:06:43.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:06:43.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:06:43.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:43.147939 18269 mini_oidc.cc:150] Starting JWKS server
15:06:43.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:43.151343 18269 webserver.cc:492] Webserver started at http://127.0.0.1:40871/ using document root <none> and password file <none>
15:06:43.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:43.151844 18269 mini_oidc.cc:186] Starting OIDC Discovery server
15:06:43.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:43.153584 18269 webserver.cc:492] Webserver started at http://127.0.0.1:42665/ using document root <none> and password file <none>
15:06:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:06:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/principal' for realm 'KRBTEST.COM',
15:06:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:06:43.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:43 dist-test-slave-xht5 krb5kdc[18281](info): setting up network...
15:06:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:06:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:43 dist-test-slave-xht5 krb5kdc[18281](info): set up 2 sockets
15:06:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:43 dist-test-slave-xht5 krb5kdc[18281](info): commencing operation
15:06:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:06:43.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:43.447822 18259 heartbeater.cc:499] Master 127.17.34.62:44789 was elected leader, sending a full tablet report...
15:06:45.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.260494 18269 mini_kdc.cc:121] Time spent starting KDC: real 2.066s user 0.003s sys 0.016s
15:06:45.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:06:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:06:45.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:06:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:06:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:06:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:06:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:06:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:06:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:06:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:06:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/test-user.keytab.
15:06:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/test-user.keytab.
15:06:45.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:45 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638405, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:45.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:06:45.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:06:45Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:06:45.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:06:45Z Disabled control of system clock
15:06:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.126@KRBTEST.COM; defaulting to no policy
15:06:45.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:45.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.126@KRBTEST.COM" created.
15:06:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:45.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.126@KRBTEST.COM; defaulting to no policy
15:06:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.126@KRBTEST.COM" created.
15:06:45.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:45.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:45.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:45.507272 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:45.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:45.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:06:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.816756 18298 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:45.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.817312 18298 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:45.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.817994 18298 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:45.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.850505 18298 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:45.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.850744 18298 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.851006 18298 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.851203 18298 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.851397 18298 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.851601 18298 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.851748 18298 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:45.889868 18298 master_runner.cc:386] Master server non-default flags:
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:45.891160 18298 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:45.892163 18298 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.904675 18305 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:47.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:45.905246 18304 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:47.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.213153 18307 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:47.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.216228 18306 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1307 milliseconds
15:06:47.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.216341 18298 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:47.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.217696 18298 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:47.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.220376 18298 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:47.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.221756 18298 hybrid_clock.cc:648] HybridClock initialized: now 1765638407221710 us; error 52 us; skew 500 ppm
15:06:47.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.222728 18298 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:47 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638407, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.228960 18298 init.cc:377] Logged in from keytab as kudu/127.17.215.126@KRBTEST.COM (short username kudu)
15:06:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.234943 18298 webserver.cc:492] Webserver started at http://127.17.215.126:38781/ using document root <none> and password file <none>
15:06:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.236145 18298 fs_manager.cc:362] Metadata directory not provided
15:06:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.236414 18298 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.236950 18298 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.243021 18298 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/instance:
15:06:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "deb58b8805af4e5abe3265e852364b34"
15:06:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:47 on dist-test-slave-xht5"
15:06:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.244500 18298 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal/instance:
15:06:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "deb58b8805af4e5abe3265e852364b34"
15:06:47.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:47 on dist-test-slave-xht5"
15:06:47.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.254056 18298 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
15:06:47.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.261118 18316 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:47.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.262390 18298 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.004s
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.262823 18298 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "deb58b8805af4e5abe3265e852364b34"
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:47 on dist-test-slave-xht5"
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.263247 18298 fs_report.cc:389] FS layout report
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/data
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:47.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.320425 18298 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:47.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.326143 18298 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:47.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.326609 18298 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:47.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.407490 18298 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.126:37719
15:06:47.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.407573 18367 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.126:37719 every 8 connection(s)
15:06:47.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.410166 18298 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:06:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.415879 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18298
15:06:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.416395 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal/instance
15:06:47.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.418272 18368 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:47.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.431831 18368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
15:06:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.125@KRBTEST.COM; defaulting to no policy
15:06:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:47.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.125@KRBTEST.COM" created.
15:06:47.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:47 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638407, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:47.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:47.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:47.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:47.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.475312 18368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } has no permanent_uuid. Determining permanent_uuid...
15:06:47.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.478050 18319 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.483354 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.125@KRBTEST.COM; defaulting to no policy
15:06:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.125@KRBTEST.COM" created.
15:06:47.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:47.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:47.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.510207 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:47.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:06:47.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:47.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.532866 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 1
15:06:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.537622 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.606076 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 2
15:06:47.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.609841 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.701289 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 3
15:06:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.705770 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.833189 18377 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:47.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.833766 18377 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:47.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.834401 18377 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.849200 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 4
15:06:47.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.852924 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:47.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.867666 18377 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.867904 18377 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.868172 18377 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.868367 18377 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.868571 18377 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.868764 18377 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:47.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.868988 18377 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.907760 18377 master_runner.cc:386] Master server non-default flags:
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:06:47.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.909085 18377 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:47.910496 18377 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.924736 18383 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:48.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:48.152320 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 5
15:06:48.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:48.160209 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:48.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:48.707823 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 6
15:06:48.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:48.739179 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.327898 18382 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 18377
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.742656 18382 kernel_stack_watchdog.cc:198] Thread 18377 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:06:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:47.924769 18384 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.743880 18377 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.819s user 0.689s sys 1.008s
15:06:49.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.744879 18377 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.820s user 0.689s sys 1.008s
15:06:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.747454 18386 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:49.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.749432 18385 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1820 milliseconds
15:06:49.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.749440 18377 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:49.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.751029 18377 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:49.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.753502 18377 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.754859 18377 hybrid_clock.cc:648] HybridClock initialized: now 1765638409754811 us; error 48 us; skew 500 ppm
15:06:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.755749 18377 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:49.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:49 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638409, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.762101 18377 init.cc:377] Logged in from keytab as kudu/127.17.215.125@KRBTEST.COM (short username kudu)
15:06:49.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.768021 18377 webserver.cc:492] Webserver started at http://127.17.215.125:37333/ using document root <none> and password file <none>
15:06:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.768988 18377 fs_manager.cc:362] Metadata directory not provided
15:06:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.769203 18377 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:49.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.769701 18377 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:49.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.774446 18377 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/instance:
15:06:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:06:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:49 on dist-test-slave-xht5"
15:06:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.775599 18377 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal/instance:
15:06:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:06:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:49 on dist-test-slave-xht5"
15:06:49.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.783560 18377 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
15:06:49.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.789408 18399 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:49.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.790539 18377 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.790823 18377 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:49 on dist-test-slave-xht5"
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.791113 18377 fs_report.cc:389] FS layout report
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/data
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.799750 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 7
15:06:49.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:49.807058 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.125:38877: Network error: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:06:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.858464 18377 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.867007 18377 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:49.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.867561 18377 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.949342 18377 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.125:38877
15:06:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.949397 18452 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.125:38877 every 8 connection(s)
15:06:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.952209 18377 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:06:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.958953 18453 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.961385 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18377
15:06:49.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.961826 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal/instance
15:06:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:49.970464 18453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
15:06:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.124@KRBTEST.COM; defaulting to no policy
15:06:49.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:49.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.124@KRBTEST.COM" created.
15:06:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:49 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638409, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:50.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.008983 18453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } has no permanent_uuid. Determining permanent_uuid...
15:06:50.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:50 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638409, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.124@KRBTEST.COM; defaulting to no policy
15:06:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.124@KRBTEST.COM" created.
15:06:50.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.035676 18453 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } has no permanent_uuid. Determining permanent_uuid...
15:06:50.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.038611 18402 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.043174 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:50.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:50.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.052588 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:06:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:50.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.092739 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 1
15:06:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.096504 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.164883 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 2
15:06:50.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.169998 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.261442 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 3
15:06:50.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.265777 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.382673 18463 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.383318 18463 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:50.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.384168 18463 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:50.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.409197 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 4
15:06:50.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.412729 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.429816 18463 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:06:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.430140 18463 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:06:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.430640 18463 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.430976 18463 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:06:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.431281 18463 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:06:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.431581 18463 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:06:50.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.431815 18463 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:06:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.471431 18463 master_runner.cc:386] Master server non-default flags:
15:06:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:50.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.472729 18463 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:50.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.473706 18463 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.488065 18469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:50.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:50.712096 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 5
15:06:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.715714 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:51.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.263134 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 6
15:06:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:51.296245 18453 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:51.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:50.489614 18470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:51.793357 18472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:51.797544 18471 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1306 milliseconds
15:06:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.797657 18463 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.799306 18463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:51.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.802501 18463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.803931 18463 hybrid_clock.cc:648] HybridClock initialized: now 1765638411803872 us; error 79 us; skew 500 ppm
15:06:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.804735 18463 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:51 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638411, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:51.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.811187 18463 init.cc:377] Logged in from keytab as kudu/127.17.215.124@KRBTEST.COM (short username kudu)
15:06:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.817472 18463 webserver.cc:492] Webserver started at http://127.17.215.124:45669/ using document root <none> and password file <none>
15:06:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.818538 18463 fs_manager.cc:362] Metadata directory not provided
15:06:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.818751 18463 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.819242 18463 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.823951 18463 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/instance:
15:06:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ca3bf63db54016822bfee8b1669160"
15:06:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:51 on dist-test-slave-xht5"
15:06:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.825016 18463 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal/instance:
15:06:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ca3bf63db54016822bfee8b1669160"
15:06:51.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:51 on dist-test-slave-xht5"
15:06:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.832722 18463 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:06:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.838969 18484 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.840134 18463 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
15:06:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.840488 18463 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:06:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ca3bf63db54016822bfee8b1669160"
15:06:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:51 on dist-test-slave-xht5"
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.840814 18463 fs_report.cc:389] FS layout report
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/data
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.897436 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } attempt: 8
15:06:51.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.901726 18463 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:51 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638407, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.909438 18463 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.909837 18463 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.921983 18368 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } has no permanent_uuid. Determining permanent_uuid...
15:06:51.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:51.928781 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:51.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:51.994161 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 1
15:06:52.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.000118 18368 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.17.215.124:36397: Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:06:52.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.007403 18463 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.124:36397
15:06:52.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.007462 18538 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.124:36397 every 8 connection(s)
15:06:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.009920 18463 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:06:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.017047 18539 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:06:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.017213 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18463
15:06:52.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.017676 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal/instance
15:06:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.030126 18539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } has no permanent_uuid. Determining permanent_uuid...
15:06:52.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638405, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:52.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.053598 18368 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 2
15:06:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638411, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638407, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:52.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.076538 18539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } has no permanent_uuid. Determining permanent_uuid...
15:06:52.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638405, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:52.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638411, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:52.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.095678 18368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap starting.
15:06:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638405, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:52.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.102779 18368 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Neither blocks nor log segments found. Creating new log.
15:06:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.105614 18368 log.cc:826] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Log is configured to *not* fsync() on all Append() calls
15:06:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.107481 18539 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } has no permanent_uuid. Determining permanent_uuid...
15:06:52.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.117535 18368 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: No bootstrap required, opened a new log
15:06:52.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638411, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.139523 18539 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap starting.
15:06:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.141026 18368 raft_consensus.cc:359] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.141841 18368 raft_consensus.cc:385] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.142122 18368 raft_consensus.cc:740] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: deb58b8805af4e5abe3265e852364b34, State: Initialized, Role: FOLLOWER
15:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.142843 18368 consensus_queue.cc:260] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.146811 18549 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.147666 18549 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.149204 18539 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Neither blocks nor log segments found. Creating new log.
15:06:52.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.149475 18368 sys_catalog.cc:565] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: configured and running, proceeding with master startup.
15:06:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.152000 18549 raft_consensus.cc:493] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:06:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.152410 18549 raft_consensus.cc:515] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.156498 18549 leader_election.cc:290] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fc71b6722f04408abf55d751c7f83cbb (127.17.215.125:38877), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:06:52.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.157228 18539 log.cc:826] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Log is configured to *not* fsync() on all Append() calls
15:06:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.168687 18539 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: No bootstrap required, opened a new log
15:06:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.198359 18539 raft_consensus.cc:359] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.199465 18539 raft_consensus.cc:385] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.199849 18539 raft_consensus.cc:740] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87ca3bf63db54016822bfee8b1669160, State: Initialized, Role: FOLLOWER
15:06:52.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.201481 18539 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.211958 18539 sys_catalog.cc:565] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: configured and running, proceeding with master startup.
15:06:52.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.212209 18514 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "deb58b8805af4e5abe3265e852364b34" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160" is_pre_election: true
15:06:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.213181 18514 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate deb58b8805af4e5abe3265e852364b34 in term 0.
15:06:52.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.215019 18317 leader_election.cc:304] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, deb58b8805af4e5abe3265e852364b34; no voters:
15:06:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.215768 18549 raft_consensus.cc:2804] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:06:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.210680 18562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.216122 18549 raft_consensus.cc:493] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:06:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.216450 18549 raft_consensus.cc:3060] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 0 FOLLOWER]: Advancing to term 1
15:06:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.216528 18562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.227828 18549 raft_consensus.cc:515] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.233955 18514 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "deb58b8805af4e5abe3265e852364b34" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160"
15:06:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.234577 18514 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 0 FOLLOWER]: Advancing to term 1
15:06:52.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.231910 18549 leader_election.cc:290] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [CANDIDATE]: Term 1 election: Requested vote from peers fc71b6722f04408abf55d751c7f83cbb (127.17.215.125:38877), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:06:52.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.250653 18563 catalog_manager.cc:1568] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:52.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.250986 18563 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:52.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.242365 18514 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate deb58b8805af4e5abe3265e852364b34 in term 1.
15:06:52.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.257843 18317 leader_election.cc:304] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, deb58b8805af4e5abe3265e852364b34; no voters:
15:06:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.258667 18549 raft_consensus.cc:2804] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 FOLLOWER]: Leader election won for term 1
15:06:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.259130 18549 raft_consensus.cc:697] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 LEADER]: Becoming Leader. State: Replica: deb58b8805af4e5abe3265e852364b34, State: Running, Role: LEADER
15:06:52.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.260111 18549 consensus_queue.cc:237] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.279907 18550 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: New leader deb58b8805af4e5abe3265e852364b34. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.283308 18550 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: LEADER
15:06:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.247762 18298 master_runner.cc:186] Error getting master registration for 127.17.215.125:38877: OK, instance_id { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" instance_seqno: 1765638409927134 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.316394 18298 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.329528 18577 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:06:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.330250 18577 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:06:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.337345 18322 tablet.cc:2378] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:52.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.340741 18575 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:06:52.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.356772 18453 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } attempt: 7
15:06:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.360590 18575 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:06:52.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:52 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638409, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.415591 18514 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Refusing update from remote peer deb58b8805af4e5abe3265e852364b34: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.416723 18492 tablet.cc:2378] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:52.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.419008 18453 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap starting.
15:06:52.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.420044 18549 consensus_queue.cc:1048] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [LEADER]: Connected to new peer: Peer: permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.435410 18453 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Neither blocks nor log segments found. Creating new log.
15:06:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.438112 18453 log.cc:826] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Log is configured to *not* fsync() on all Append() calls
15:06:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.315317 18463 master_runner.cc:186] Error getting master registration for 127.17.215.125:38877: OK, instance_id { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" instance_seqno: 1765638409927134 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:06:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.453522 18463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:06:52.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.454739 18453 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: No bootstrap required, opened a new log
15:06:52.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.480278 18562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: New leader deb58b8805af4e5abe3265e852364b34. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.481009 18562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.489187 18580 mvcc.cc:204] Tried to move back new op lower bound from 7232054937194315776 to 7232054936637845504. Current Snapshot: MvccSnapshot[applied={T|T < 7232054937194315776}]
15:06:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.509644 18581 mvcc.cc:204] Tried to move back new op lower bound from 7232054937194315776 to 7232054936637845504. Current Snapshot: MvccSnapshot[applied={T|T < 7232054937194315776}]
15:06:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.508608 18550 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.513326 18550 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: LEADER
15:06:52.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.517712 18562 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.518383 18562 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.526889 18575 catalog_manager.cc:1357] Generated new cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:06:52.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.527208 18575 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:06:52.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.551168 18453 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.552286 18453 raft_consensus.cc:385] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:06:52.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.555212 18453 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc71b6722f04408abf55d751c7f83cbb, State: Initialized, Role: FOLLOWER
15:06:52.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.565945 18575 catalog_manager.cc:1380] Generated new certificate authority record
15:06:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.568598 18575 catalog_manager.cc:1514] Loading token signing keys...
15:06:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.566326 18453 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:06:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.571875 18586 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.573060 18586 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.581941 18453 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: configured and running, proceeding with master startup.
15:06:52.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.583915 18426 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 0 FOLLOWER]: Advancing to term 1
15:06:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.582094 18427 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "deb58b8805af4e5abe3265e852364b34" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:06:52.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.589582 18428 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "deb58b8805af4e5abe3265e852364b34" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fc71b6722f04408abf55d751c7f83cbb" is_pre_election: true
15:06:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.594986 18426 raft_consensus.cc:1275] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Refusing update from remote peer deb58b8805af4e5abe3265e852364b34: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:06:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.597270 18550 consensus_queue.cc:1048] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:06:52.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.610532 18575 catalog_manager.cc:6027] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Generated new TSK 0
15:06:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.611912 18575 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:06:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.626463 18407 tablet.cc:2378] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:06:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.695945 18586 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: New leader deb58b8805af4e5abe3265e852364b34. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.696110 18593 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:52.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.697026 18593 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.697078 18586 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: FOLLOWER
15:06:52.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.65@KRBTEST.COM; defaulting to no policy
15:06:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:52.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.65@KRBTEST.COM" created.
15:06:52.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:52.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:52.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:52.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:52.758623 18269 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.17.215.65: real 0.113s user 0.002s sys 0.018s
15:06:52.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.765709 18604 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:06:52.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.767724 18604 catalog_manager.cc:1562] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: loading cluster ID for follower catalog manager: success
15:06:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.65@KRBTEST.COM; defaulting to no policy
15:06:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:52.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.65@KRBTEST.COM" created.
15:06:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.792505 18604 catalog_manager.cc:1584] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: acquiring CA information for follower catalog manager: success
15:06:52.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.798424 18604 catalog_manager.cc:1612] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:52.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:52.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:52.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.822973 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/logs
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data/info.pb
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.65:0
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.65
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.65
15:06:52.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.65
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:52.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.999070 18550 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "deb58b8805af4e5abe3265e852364b34" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:06:53.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:52.999876 18550 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: LEADER
15:06:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.336333 18577 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:06:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.336627 18577 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: loading cluster ID for follower catalog manager: success
15:06:53.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.345124 18577 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: acquiring CA information for follower catalog manager: success
15:06:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.361505 18577 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:06:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.409988 18610 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.410964 18610 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.412181 18610 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:53.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.477489 18610 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:53.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.479570 18610 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.215.65
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.564901 18610 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data
15:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.65
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.65:0
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data/info.pb
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.65
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.65
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/logs
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.569482 18610 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:53.571101 18610 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:53.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.604287 18622 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:54.997131 18620 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 18610
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:55.667508 18620 kernel_stack_watchdog.cc:198] Thread 18610 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 386ms:
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:06:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:53.606462 18621 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:55.675802 18610 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.068s user 0.003s sys 0.001s
15:06:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:55.676226 18610 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.068s user 0.003s sys 0.001s
15:06:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:55.679145 18624 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.685371 18610 server_base.cc:1047] running on GCE node
15:06:55.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.687235 18610 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:55.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.699009 18610 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:55.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.700840 18610 hybrid_clock.cc:648] HybridClock initialized: now 1765638415700737 us; error 113 us; skew 500 ppm
15:06:55.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.702178 18610 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:55.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:55 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638415, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:55.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.710647 18610 init.cc:377] Logged in from keytab as kudu/127.17.215.65@KRBTEST.COM (short username kudu)
15:06:55.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.734084 18610 webserver.cc:492] Webserver started at http://127.17.215.65:44033/ using document root <none> and password file <none>
15:06:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.736037 18610 fs_manager.cc:362] Metadata directory not provided
15:06:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.736543 18610 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:55.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.737257 18610 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.744781 18610 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data/instance:
15:06:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95b520c6a290469daf2e1a60c0a06541"
15:06:55.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:55 on dist-test-slave-xht5"
15:06:55.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.746662 18610 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal/instance:
15:06:55.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95b520c6a290469daf2e1a60c0a06541"
15:06:55.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:55 on dist-test-slave-xht5"
15:06:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.764523 18610 fs_manager.cc:696] Time spent creating directory manager: real 0.017s user 0.005s sys 0.009s
15:06:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.778244 18637 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:55.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.780074 18610 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.005s sys 0.004s
15:06:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.780686 18610 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal
15:06:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "95b520c6a290469daf2e1a60c0a06541"
15:06:55.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:55 on dist-test-slave-xht5"
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.781406 18610 fs_report.cc:389] FS layout report
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data/data
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.917336 18610 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.930538 18610 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.931334 18610 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.940907 18610 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.953584 18610 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.954133 18610 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
15:06:55.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.954643 18610 ts_tablet_manager.cc:616] Registered 0 tablets
15:06:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:55.955070 18610 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.558820 18610 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.65:42129
15:06:56.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.558925 18750 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.65:42129 every 8 connection(s)
15:06:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.564006 18610 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/data/info.pb
15:06:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.574100 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18610
15:06:56.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.574818 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-0/wal/instance
15:06:56.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.66@KRBTEST.COM; defaulting to no policy
15:06:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:56 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638415, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:56 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638415, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.66@KRBTEST.COM" created.
15:06:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:56 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638415, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:56.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.688632 18753 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:06:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.689160 18753 heartbeater.cc:461] Registering TS with master...
15:06:56.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.690799 18753 heartbeater.cc:507] Master 127.17.215.126:37719 requested a full tablet report, sending...
15:06:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.694583 18333 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:06:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.697504 18333 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:35943
15:06:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.698004 18752 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:06:56.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.698372 18752 heartbeater.cc:461] Registering TS with master...
15:06:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.699445 18752 heartbeater.cc:507] Master 127.17.215.124:36397 requested a full tablet report, sending...
15:06:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.703471 18504 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:06:56.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.707296 18751 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:06:56.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.707579 18751 heartbeater.cc:461] Registering TS with master...
15:06:56.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.708112 18751 heartbeater.cc:507] Master 127.17.215.125:38877 requested a full tablet report, sending...
15:06:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.712459 18418 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:06:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:56.713398 18269 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.17.215.66: real 0.122s user 0.004s sys 0.011s
15:06:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.66@KRBTEST.COM; defaulting to no policy
15:06:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.66@KRBTEST.COM" created.
15:06:56.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:56.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:56.758751 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/logs
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data/info.pb
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.66:0
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.66
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.66
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.66
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.105989 18765 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.106534 18765 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.107260 18765 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:57.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.140444 18765 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:57.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.141825 18765 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.215.66
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:57.180038 18765 tablet_server_runner.cc:78] Tablet server non-default flags:
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.66
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.66:0
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data/info.pb
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.66
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:06:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.66
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/logs
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:06:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:57.181571 18765 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:06:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:57.182626 18765 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:06:57.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.196872 18772 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:57 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638417, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:57.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:57.235157 18312 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:57.704782 18753 heartbeater.cc:499] Master 127.17.215.126:37719 was elected leader, sending a full tablet report...
15:06:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:58.599496 18771 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 18765
15:06:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.106984 18774 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1907 milliseconds
15:06:59.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.106058 18765 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.908s user 0.002s sys 0.000s
15:06:59.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.107371 18765 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.910s user 0.002s sys 0.000s
15:06:59.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:57.198032 18773 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.108826 18775 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:06:59.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.108773 18765 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:06:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.112767 18765 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:06:59.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.115388 18765 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:06:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.116900 18765 hybrid_clock.cc:648] HybridClock initialized: now 1765638419116846 us; error 69 us; skew 500 ppm
15:06:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.117623 18765 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:06:59.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:59 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638419, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:59.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.124573 18765 init.cc:377] Logged in from keytab as kudu/127.17.215.66@KRBTEST.COM (short username kudu)
15:06:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.129630 18765 webserver.cc:492] Webserver started at http://127.17.215.66:45659/ using document root <none> and password file <none>
15:06:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.130708 18765 fs_manager.cc:362] Metadata directory not provided
15:06:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.130990 18765 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:06:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.131567 18765 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:06:59.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.136502 18765 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data/instance:
15:06:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e16d92744730461090a62bf1861a9bd9"
15:06:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:59 on dist-test-slave-xht5"
15:06:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.137573 18765 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal/instance:
15:06:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e16d92744730461090a62bf1861a9bd9"
15:06:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:59 on dist-test-slave-xht5"
15:06:59.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.144747 18765 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
15:06:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.150017 18787 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:06:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.151202 18765 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:06:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.151494 18765 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal
15:06:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e16d92744730461090a62bf1861a9bd9"
15:06:59.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:59 on dist-test-slave-xht5"
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.151793 18765 fs_report.cc:389] FS layout report
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data/data
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:06:59.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:06:59.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.210994 18765 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:06:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.216710 18765 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:06:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.217156 18765 kserver.cc:163] Server-wide thread pool size limit: 3276
15:06:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.219586 18765 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:06:59.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.224187 18765 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:06:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.224387 18765 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:06:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.224593 18765 ts_tablet_manager.cc:616] Registered 0 tablets
15:06:59.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.224714 18765 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:06:59.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.414397 18765 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.66:43305
15:06:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.415076 18900 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.66:43305 every 8 connection(s)
15:06:59.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.418063 18765 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/data/info.pb
15:06:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.425789 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18765
15:06:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.426254 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-1/wal/instance
15:06:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:59 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638419, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:06:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:59 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638419, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:06:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.494738 18902 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:06:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.495414 18902 heartbeater.cc:461] Registering TS with master...
15:06:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.498081 18902 heartbeater.cc:507] Master 127.17.215.124:36397 requested a full tablet report, sending...
15:06:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.502786 18504 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:06:59.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:59 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638419, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:06:59.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.215.67@KRBTEST.COM; defaulting to no policy
15:06:59.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.519985 18901 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:06:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.520547 18901 heartbeater.cc:461] Registering TS with master...
15:06:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.521394 18901 heartbeater.cc:507] Master 127.17.215.125:38877 requested a full tablet report, sending...
15:06:59.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:59.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.215.67@KRBTEST.COM" created.
15:06:59.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.523933 18418 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:06:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.529349 18907 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:06:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.529736 18907 heartbeater.cc:461] Registering TS with master...
15:06:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.530805 18907 heartbeater.cc:507] Master 127.17.215.126:37719 requested a full tablet report, sending...
15:06:59.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.533245 18333 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:06:59.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.534574 18333 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:59979
15:06:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.215.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:59.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.547258 18269 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.17.215.67: real 0.105s user 0.005s sys 0.009s
15:06:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.17.215.67@KRBTEST.COM; defaulting to no policy
15:06:59.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.17.215.67@KRBTEST.COM" created.
15:06:59.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:06:59.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:59.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.17.215.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab.
15:06:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.589170 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/logs
15:06:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data/info.pb
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.67:0
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.67
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.67
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.67
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:06:59.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:06:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:06:59 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638419, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:06:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:06:59.767215 18395 init.cc:295] Successfully reacquired a new kerberos TGT
15:06:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.933990 18915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:06:59.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.934587 18915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:06:59.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.935319 18915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:06:59.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.969929 18915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:06:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:06:59.971063 18915 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.215.67
15:07:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:00.013383 18915 tablet_server_runner.cc:78] Tablet server non-default flags:
15:07:00.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.67
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.67:0
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data/info.pb
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.67
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.215.67
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/logs
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:00.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:00.015115 18915 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:00.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:00.016463 18915 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:00.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:00.032027 18921 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:00.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:00.537459 18907 heartbeater.cc:499] Master 127.17.215.126:37719 was elected leader, sending a full tablet report...
15:07:01.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:00.038091 18924 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.389859 18915 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:07:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:00.037354 18922 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:01.390508 18923 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1354 milliseconds
15:07:01.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.395188 18915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.398123 18915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.399818 18915 hybrid_clock.cc:648] HybridClock initialized: now 1765638421399739 us; error 93 us; skew 500 ppm
15:07:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.400909 18915 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:01.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.408047 18915 init.cc:377] Logged in from keytab as kudu/127.17.215.67@KRBTEST.COM (short username kudu)
15:07:01.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.415428 18915 webserver.cc:492] Webserver started at http://127.17.215.67:42947/ using document root <none> and password file <none>
15:07:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.416491 18915 fs_manager.cc:362] Metadata directory not provided
15:07:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.416741 18915 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.417251 18915 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:07:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.422489 18915 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data/instance:
15:07:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbc5717d67e140648f9ce018c4ce3c15"
15:07:01.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:07:01 on dist-test-slave-xht5"
15:07:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.424347 18915 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal/instance:
15:07:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbc5717d67e140648f9ce018c4ce3c15"
15:07:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:07:01 on dist-test-slave-xht5"
15:07:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.435120 18915 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
15:07:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.445405 18935 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.446939 18915 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.006s sys 0.001s
15:07:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.447499 18915 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbc5717d67e140648f9ce018c4ce3c15"
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:07:01 on dist-test-slave-xht5"
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.447914 18915 fs_report.cc:389] FS layout report
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal
15:07:01.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data/data
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:01.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.504442 18915 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.516147 18915 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.516566 18915 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.519479 18915 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:07:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.524159 18915 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:07:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.524377 18915 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:07:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.524569 18915 ts_tablet_manager.cc:616] Registered 0 tablets
15:07:01.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.524709 18915 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:07:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.723006 18915 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.67:34603
15:07:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.723491 19047 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.67:34603 every 8 connection(s)
15:07:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.725905 18915 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/data/info.pb
15:07:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.728966 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 18915
15:07:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.729434 18269 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/ts-2/wal/instance
15:07:01.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:07:01.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:01.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:07:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.821870 18480 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.824370 19048 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:01.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.824643 19052 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:07:01.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.824980 19048 heartbeater.cc:461] Registering TS with master...
15:07:01.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.825193 19052 heartbeater.cc:461] Registering TS with master...
15:07:01.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.827651 19048 heartbeater.cc:507] Master 127.17.215.125:38877 requested a full tablet report, sending...
15:07:01.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.827726 19052 heartbeater.cc:507] Master 127.17.215.126:37719 requested a full tablet report, sending...
15:07:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.832572 18418 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.833496 19049 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.833643 18333 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.833961 19049 heartbeater.cc:461] Registering TS with master...
15:07:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.835676 19049 heartbeater.cc:507] Master 127.17.215.124:36397 requested a full tablet report, sending...
15:07:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.836359 18333 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:37893
15:07:01.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.840278 18504 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:01.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:01.856768 18269 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:07:01.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:01.878 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:07:01.890 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:07:01.891 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:07:01.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.215.126"
port: 37719
}
bound_http_address {
host: "127.17.215.126"
port: 38781
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.215.125"
port: 38877
}
bound_http_address {
host: "127.17.215.125"
port: 37333
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.215.124"
port: 36397
}
bound_http_address {
host: "127.17.215.124"
port: 45669
}
}
}
15:07:01.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:07:01.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.215.65"
port: 42129
}
bound_http_address {
host: "127.17.215.65"
port: 44033
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.215.66"
port: 43305
}
bound_http_address {
host: "127.17.215.66"
port: 45659
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.215.67"
port: 34603
}
bound_http_address {
host: "127.17.215.67"
port: 42947
}
}
}
15:07:01.896 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:07:01.897 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:07:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:01 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638421, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:01.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:07:01.949 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:01.952 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc
15:07:01.960 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:01.961 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc
15:07:01.963 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:01.967 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:07:01.969 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:07:01.969 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:07:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:02 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638422, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:02.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:07:02.019 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:02.020 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc
15:07:02.021 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:02.022 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.125:38877
15:07:02.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:07:02.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.024500 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18377
15:07:02.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:02.059 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.124:36397
15:07:02.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:07:02.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.060200 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18463
15:07:02.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:02.095 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.126:37719
15:07:02.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:07:02.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.096172 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18298
15:07:02.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:02.130 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.125:38877
15:07:02.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:07:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.135362 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:07:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37333
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:02.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:02.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:02.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:02.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.462097 19063 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:02.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.462680 19063 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:02.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.463284 19063 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.496945 19063 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.497170 19063 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.497423 19063 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.497641 19063 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.497835 19063 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.498023 19063 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:02.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.498178 19063 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:02.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.519572 18902 heartbeater.cc:646] Failed to heartbeat to 127.17.215.124:36397 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:07:02.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.541564 19063 master_runner.cc:386] Master server non-default flags:
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37333
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:02.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.542966 19063 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:02.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.543943 19063 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.556046 19070 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.757093 18751 heartbeater.cc:646] Failed to heartbeat to 127.17.215.125:38877 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:07:02.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:02.841722 19052 heartbeater.cc:499] Master 127.17.215.126:37719 was elected leader, sending a full tablet report...
15:07:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.844276 19048 heartbeater.cc:646] Failed to heartbeat to 127.17.215.125:38877 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.17.215.125:38877: connect: Connection refused (error 111)
15:07:03.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:02.558650 19071 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:03.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:03.871834 19072 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:07:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:03.876008 19063 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.319s user 0.507s sys 0.797s
15:07:03.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:03.876423 19063 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.320s user 0.507s sys 0.798s
15:07:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:03.877482 19073 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.877529 19063 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:07:03.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.879280 19063 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:03.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.882581 19063 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:03.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.884055 19063 hybrid_clock.cc:648] HybridClock initialized: now 1765638423884011 us; error 35 us; skew 500 ppm
15:07:03.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.885290 19063 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:03.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:03 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638423, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.894764 19063 init.cc:377] Logged in from keytab as kudu/127.17.215.125@KRBTEST.COM (short username kudu)
15:07:03.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.901106 19063 webserver.cc:492] Webserver started at http://127.17.215.125:37333/ using document root <none> and password file <none>
15:07:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.902390 19063 fs_manager.cc:362] Metadata directory not provided
15:07:03.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.902738 19063 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:03.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.914551 19063 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.002s
15:07:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.920703 19093 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.922173 19063 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
15:07:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.922644 19063 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:03.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:49 on dist-test-slave-xht5"
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:03.925637 19063 fs_report.cc:389] FS layout report
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/data
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:03.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:04.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.004906 19063 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.010704 19063 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.011111 19063 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:04.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.083005 19063 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.125:38877
15:07:04.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.083070 19144 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.125:38877 every 8 connection(s)
15:07:04.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.085871 19063 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.094287 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19063
15:07:04.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:04.095 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.124:36397
15:07:04.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.096905 19145 sys_catalog.cc:263] Verifying existing consensus state
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.097065 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:04.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45669
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:04.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:04.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:04.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.102365 19145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap starting.
15:07:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.145732 19145 log.cc:826] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Log is configured to *not* fsync() on all Append() calls
15:07:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.160813 19145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.161552 19145 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap complete.
15:07:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.183888 19145 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.186331 19145 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc71b6722f04408abf55d751c7f83cbb, State: Initialized, Role: FOLLOWER
15:07:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.187209 19145 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:04.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.190004 19149 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:04.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.190837 19149 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: FOLLOWER
15:07:04.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.191744 19145 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: configured and running, proceeding with master startup.
15:07:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.223421 19160 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:04.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.223984 19160 catalog_manager.cc:1562] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: loading cluster ID for follower catalog manager: success
15:07:04.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.231760 19160 catalog_manager.cc:1584] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: acquiring CA information for follower catalog manager: success
15:07:04.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.238122 19160 catalog_manager.cc:1612] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.218138 19063 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.254953 19063 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:04.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.458784 19147 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:04.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.459542 19147 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:04.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.460601 19147 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.495570 19147 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.495820 19147 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.496086 19147 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.496285 19147 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:04.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.496528 19147 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.496732 19147 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:04.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.496873 19147 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.544850 19147 master_runner.cc:386] Master server non-default flags:
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45669
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.546500 19147 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:04.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.547926 19147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.591832 18901 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:04.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.601888 19110 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e16d92744730461090a62bf1861a9bd9" instance_seqno: 1765638419369887) as {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:51181; Asking this server to re-register.
15:07:04.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.603983 18901 heartbeater.cc:461] Registering TS with master...
15:07:04.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.608095 19109 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:07:04.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.612042 19171 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:04.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.783159 18751 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.787412 19110 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "95b520c6a290469daf2e1a60c0a06541" instance_seqno: 1765638416377906) as {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:59733; Asking this server to re-register.
15:07:04.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.789335 18751 heartbeater.cc:461] Registering TS with master...
15:07:04.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.793052 19110 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:07:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.922053 19048 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:04.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.925431 19110 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "fbc5717d67e140648f9ce018c4ce3c15" instance_seqno: 1765638421668181) as {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:37605; Asking this server to re-register.
15:07:04.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.927536 19048 heartbeater.cc:461] Registering TS with master...
15:07:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:04.930879 19110 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:05.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.710433 19185 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:05.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.711983 19185 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:05.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:05 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638425, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:05.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.721732 18633 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:05.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.744089 19185 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:05.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:05.752478 19095 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:05.765559 19095 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:05.774511 19094 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397): Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:07:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.775122 19094 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fc71b6722f04408abf55d751c7f83cbb; no voters: 87ca3bf63db54016822bfee8b1669160, deb58b8805af4e5abe3265e852364b34
15:07:05.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:05.776594 19185 raft_consensus.cc:2749] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:07:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:06.010885 19170 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 19147
15:07:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:06.747956 19147 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.126s user 0.002s sys 0.000s
15:07:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:06.749024 19173 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2123 milliseconds
15:07:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:06.749281 19147 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.127s user 0.003s sys 0.000s
15:07:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:04.621129 19172 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.750314 19147 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:07:06.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:06.750344 19175 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:06.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.754652 19147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.757681 19147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:06.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.759105 19147 hybrid_clock.cc:648] HybridClock initialized: now 1765638426759057 us; error 62 us; skew 500 ppm
15:07:06.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.759990 19147 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:06.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:06 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638426, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:06.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.766023 19147 init.cc:377] Logged in from keytab as kudu/127.17.215.124@KRBTEST.COM (short username kudu)
15:07:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.772125 19147 webserver.cc:492] Webserver started at http://127.17.215.124:45669/ using document root <none> and password file <none>
15:07:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.773114 19147 fs_manager.cc:362] Metadata directory not provided
15:07:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.773358 19147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:06.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.781705 19147 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
15:07:06.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.786661 19200 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:06.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.788183 19147 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:07:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.788547 19147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ca3bf63db54016822bfee8b1669160"
15:07:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:51 on dist-test-slave-xht5"
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.790576 19147 fs_report.cc:389] FS layout report
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/data
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:06.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:06.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.845825 19147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:06.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.852510 19147 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.853077 19147 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:06.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.956611 19147 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.124:36397
15:07:06.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.956724 19254 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.124:36397 every 8 connection(s)
15:07:06.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.959646 19147 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:06.962 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:06.962 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.126:37719
15:07:06.963 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:07:06.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.960778 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19147
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.967389 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:06.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38781
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:06.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:06.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:06.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:06.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.975823 19255 sys_catalog.cc:263] Verifying existing consensus state
15:07:06.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:06.984339 19255 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap starting.
15:07:07.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.040225 19255 log.cc:826] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Log is configured to *not* fsync() on all Append() calls
15:07:07.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.060578 19255 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:07.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.061609 19255 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap complete.
15:07:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.097030 19255 raft_consensus.cc:359] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.101404 19255 raft_consensus.cc:740] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87ca3bf63db54016822bfee8b1669160, State: Initialized, Role: FOLLOWER
15:07:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.103087 19255 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.115448 19260 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:07.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.116364 19260 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:07.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.120236 19255 sys_catalog.cc:565] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: configured and running, proceeding with master startup.
15:07:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.203173 19271 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:07.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.203445 19271 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: loading cluster ID for follower catalog manager: success
15:07:07.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.210610 19271 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: acquiring CA information for follower catalog manager: success
15:07:07.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.217012 19271 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.201401 19147 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.233619 19147 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.380541 19257 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:07.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.381354 19257 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:07.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.382737 19257 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.440114 19257 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.441383 19257 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.442112 19257 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.442579 19257 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.443032 19257 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.443447 19257 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.443792 19257 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.507395 19257 master_runner.cc:386] Master server non-default flags:
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38781
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.509109 19257 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:07.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.510546 19257 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.523916 19279 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:07.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.659871 18902 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.669126 19219 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e16d92744730461090a62bf1861a9bd9" instance_seqno: 1765638419369887) as {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:42017; Asking this server to re-register.
15:07:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.671175 18902 heartbeater.cc:461] Registering TS with master...
15:07:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.675554 19219 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:07:07.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.829386 18752 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.832212 19219 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "95b520c6a290469daf2e1a60c0a06541" instance_seqno: 1765638416377906) as {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:49653; Asking this server to re-register.
15:07:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.834044 18752 heartbeater.cc:461] Registering TS with master...
15:07:07.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.837023 19219 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:07:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.913499 19288 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:07.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.914330 19288 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:07.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.922734 19288 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:07.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.967494 19049 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:07.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.970845 19219 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "fbc5717d67e140648f9ce018c4ce3c15" instance_seqno: 1765638421668181) as {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:43715; Asking this server to re-register.
15:07:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.972468 19049 heartbeater.cc:461] Registering TS with master...
15:07:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.976214 19219 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:07.999594 19229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc71b6722f04408abf55d751c7f83cbb" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160" is_pre_election: true
15:07:08.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.000856 19229 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fc71b6722f04408abf55d751c7f83cbb in term 1.
15:07:08.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.003108 19094 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, fc71b6722f04408abf55d751c7f83cbb; no voters:
15:07:08.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.004181 19288 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Leader pre-election won for term 2
15:07:08.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.004544 19288 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:07:08.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.004854 19288 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 1 FOLLOWER]: Advancing to term 2
15:07:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.014971 19288 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.018085 19229 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc71b6722f04408abf55d751c7f83cbb" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160"
15:07:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.018766 19229 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 1 FOLLOWER]: Advancing to term 2
15:07:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:08.019829 19095 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.17.215.126:37719, user_credentials={real_user=slave}} blocked reactor thread for 51658.2us
15:07:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.021477 19288 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 election: Requested vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:08.024536 19095 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:08.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:08.031057 19095 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:08.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.040354 19229 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fc71b6722f04408abf55d751c7f83cbb in term 2.
15:07:08.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.041728 19094 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, fc71b6722f04408abf55d751c7f83cbb; no voters: deb58b8805af4e5abe3265e852364b34
15:07:08.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.043568 19288 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Leader election won for term 2
15:07:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.052372 19288 raft_consensus.cc:697] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 LEADER]: Becoming Leader. State: Replica: fc71b6722f04408abf55d751c7f83cbb, State: Running, Role: LEADER
15:07:08.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.053885 19288 consensus_queue.cc:237] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.070091 19295 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc71b6722f04408abf55d751c7f83cbb. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:08.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.071089 19295 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: LEADER
15:07:08.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.090826 19297 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:07:08.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.098415 19297 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:07:08.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.102130 19297 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:08.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.102391 19297 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:07:08.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.123250 19297 catalog_manager.cc:1514] Loading token signing keys...
15:07:08.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.133466 19297 catalog_manager.cc:6038] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Loaded TSK: 0
15:07:08.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.134758 19297 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:07:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.546571 19229 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Refusing update from remote peer fc71b6722f04408abf55d751c7f83cbb: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:07:08.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.548668 19295 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [LEADER]: Connected to new peer: Peer: permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:07:08.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.604950 19298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc71b6722f04408abf55d751c7f83cbb. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:08.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.609917 19298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.613184 19300 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.615199 19300 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: LEADER
15:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:08.657667 19095 consensus_peers.cc:597] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb -> Peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Couldn't send request to peer deb58b8805af4e5abe3265e852364b34. Status: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:07:08.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.667886 19298 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:08.668992 19298 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:09 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638429, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:09.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.130717 18783 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.258791 19063 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:09.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.259419 19063 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:09.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:07.530448 19280 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:08.925908 19278 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 19257
15:07:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:09.387460 19257 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.860s user 0.607s sys 1.079s
15:07:09.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:09.388983 19257 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.861s user 0.607s sys 1.079s
15:07:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:09.410769 19312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.411299 19257 server_base.cc:1047] running on GCE node
15:07:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.412434 19257 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:09.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.415309 19257 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:09.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.416821 19257 hybrid_clock.cc:648] HybridClock initialized: now 1765638429416786 us; error 40 us; skew 500 ppm
15:07:09.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.417678 19257 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:09 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638429, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.425572 19257 init.cc:377] Logged in from keytab as kudu/127.17.215.126@KRBTEST.COM (short username kudu)
15:07:09.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.435938 19257 webserver.cc:492] Webserver started at http://127.17.215.126:38781/ using document root <none> and password file <none>
15:07:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.437140 19257 fs_manager.cc:362] Metadata directory not provided
15:07:09.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.438215 19257 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.449311 19257 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.004s
15:07:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.455161 19318 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.456331 19257 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
15:07:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.456637 19257 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:09.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "deb58b8805af4e5abe3265e852364b34"
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:47 on dist-test-slave-xht5"
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.458858 19257 fs_report.cc:389] FS layout report
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/data
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:09.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:09.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.508167 19257 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.514581 19257 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.514988 19257 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.610280 19257 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.126:37719
15:07:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.610327 19369 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.126:37719 every 8 connection(s)
15:07:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.613093 19257 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:09.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.622025 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19257
15:07:09.622 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:09.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.634338 19370 sys_catalog.cc:263] Verifying existing consensus state
15:07:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.642666 19370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap starting.
15:07:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.643908 18901 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:09.687 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.215.125:38877(127.17.215.125:38877)] connecting to peer
15:07:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:09 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638429, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:09.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:09 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638423, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:09.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.792549 18907 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:07:09.799 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.215.124:36397(127.17.215.124:36397)] connecting to peer
15:07:09.802 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.215.126:37719(127.17.215.126:37719)] connecting to peer
15:07:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.818531 19370 log.cc:826] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Log is configured to *not* fsync() on all Append() calls
15:07:09.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.821540 18751 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:09.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.851227 19370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.852291 19370 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap complete.
15:07:09.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:09 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638425, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:09.888 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.17.215.124:36397(127.17.215.124:36397)] Successfully connected to peer
15:07:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.890671 19370 raft_consensus.cc:359] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.894492 19370 raft_consensus.cc:740] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: deb58b8805af4e5abe3265e852364b34, State: Initialized, Role: FOLLOWER
15:07:09.894 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.215.125:38877(127.17.215.125:38877)] Successfully connected to peer
15:07:09.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.895515 19370 consensus_queue.cc:260] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.899349 19382 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.900815 19382 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: FOLLOWER
15:07:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.903476 19370 sys_catalog.cc:565] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: configured and running, proceeding with master startup.
15:07:09.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.904140 19345 raft_consensus.cc:3060] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 1 FOLLOWER]: Advancing to term 2
15:07:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.922012 18753 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:07:09.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.957789 19048 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.982806 19389 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.987313 19389 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: FOLLOWER
15:07:09.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.988086 19382 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc71b6722f04408abf55d751c7f83cbb. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:09.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:09.988801 19382 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: FOLLOWER
15:07:10.038 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.215.126:37719(127.17.215.126:37719)] Successfully connected to peer
15:07:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.048031 19398 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.048363 19398 catalog_manager.cc:1562] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: loading cluster ID for follower catalog manager: success
15:07:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.063412 19052 heartbeater.cc:344] Connected to a master server at 127.17.215.126:37719
15:07:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.063947 19052 heartbeater.cc:499] Master 127.17.215.126:37719 was elected leader, sending a full tablet report...
15:07:10.067 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:07:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.070422 19335 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "fbc5717d67e140648f9ce018c4ce3c15" instance_seqno: 1765638421668181) as {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:42915; Asking this server to re-register.
15:07:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.071782 19052 heartbeater.cc:461] Registering TS with master...
15:07:10.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.075161 19335 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.072026 19398 catalog_manager.cc:1584] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: acquiring CA information for follower catalog manager: success
15:07:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.081971 19398 catalog_manager.cc:1612] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:10.092 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.215.125
15:07:10.094 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.215.124
15:07:10.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:10 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638429, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:10.252 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.215.126
15:07:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.285420 19300 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:10.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.288352 19300 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: LEADER
15:07:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:10 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638422, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.124@KRBTEST.COM
15:07:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:10 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638422, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.125@KRBTEST.COM
15:07:10.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:10 dist-test-slave-xht5 krb5kdc[18281](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638422, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.215.126@KRBTEST.COM
15:07:10.567 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x6c4d6220, L:/127.0.0.1:59308 - R:/127.17.215.124:36397] using SASL/GSSAPI
15:07:10.570 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x00d26b4f, L:/127.0.0.1:57022 - R:/127.17.215.125:38877] using SASL/GSSAPI
15:07:10.643 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:07:10.645 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1018), Traces: [0ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.215.125:38877, [168ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.215.124:36397, [171ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.215.126:37719, [971ms] Sub RPC ConnectToMaster: received response from server master-127.17.215.125:38877: OK, [960ms] Sub RPC ConnectToMaster: received response from server master-127.17.215.124:36397: OK, deferred=null) after lookup
15:07:10.660 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf38541ed, L:/127.0.0.1:35018 - R:/127.17.215.126:37719] using SASL/GSSAPI
15:07:10.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.802708 19335 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e16d92744730461090a62bf1861a9bd9" instance_seqno: 1765638419369887) as {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:59753; Asking this server to re-register.
15:07:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.803938 18907 heartbeater.cc:461] Registering TS with master...
15:07:10.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.806282 19335 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:07:10.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.927372 19335 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "95b520c6a290469daf2e1a60c0a06541" instance_seqno: 1765638416377906) as {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:52345; Asking this server to re-register.
15:07:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.928504 18753 heartbeater.cc:461] Registering TS with master...
15:07:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:10.931006 19335 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:11.272557 19366 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 19274
15:07:11.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:11.273207 19366 debug-util.cc:398] Leaking SignalData structure 0x7b08000717c0 after lost signal to thread 19369
15:07:11.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:11 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638431, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:11.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:11.414530 18931 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:11.685 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:07:11.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:07:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:11 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638431, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:07:11.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:11.733 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc
15:07:11.735 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:07:11.735 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.125:38877
15:07:11.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:07:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:11.736515 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19063
15:07:11.788 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.215.125:38877(127.17.215.125:38877)] handling channelInactive
15:07:11.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.215.125:38877(127.17.215.125:38877)] cleaning up while in state READY due to: connection closed
15:07:11.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:11.791 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.124:36397
15:07:11.791 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:07:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:11.792060 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19147
15:07:11.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:11.838 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.215.126:37719
15:07:11.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:07:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:11.839495 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19257
15:07:11.841 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.17.215.124:36397(127.17.215.124:36397)] handling channelInactive
15:07:11.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.215.124:36397(127.17.215.124:36397)] cleaning up while in state READY due to: connection closed
15:07:11.871 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.215.126:37719(127.17.215.126:37719)] handling channelInactive
15:07:11.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.215.126:37719(127.17.215.126:37719)] cleaning up while in state READY due to: connection closed
15:07:11.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:11.872 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.125:38877
15:07:11.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:07:11.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:11.874512 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:11.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37333
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.184942 19413 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:12.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.185459 19413 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:12.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.186077 19413 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.219099 19413 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.219331 19413 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:12.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.219615 19413 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.219873 19413 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.220082 19413 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.220273 19413 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:12.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.220419 19413 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:12.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.229077 19044 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbf40 after lost signal to thread 18916
15:07:12.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.229869 19044 debug-util.cc:398] Leaking SignalData structure 0x7b08000b54e0 after lost signal to thread 19047
15:07:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:12.260463 19413 master_runner.cc:386] Master server non-default flags:
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.125
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.125:38877
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.125
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37333
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/logs
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:12.262082 19413 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:12.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:12.263496 19413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:12.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.284916 19420 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:15.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:12.286191 19421 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.020709 19423 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:13.687999 19419 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 19413
15:07:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.018141 19413 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.732s user 0.852s sys 1.529s
15:07:15.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.021898 19413 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.736s user 0.852s sys 1.530s
15:07:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.025377 19422 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2732 milliseconds
15:07:15.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.025380 19413 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:07:15.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.026927 19413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.029474 19413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.030824 19413 hybrid_clock.cc:648] HybridClock initialized: now 1765638435030775 us; error 62 us; skew 500 ppm
15:07:15.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.031672 19413 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:15 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638435, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.038187 19413 init.cc:377] Logged in from keytab as kudu/127.17.215.125@KRBTEST.COM (short username kudu)
15:07:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.043769 19413 webserver.cc:492] Webserver started at http://127.17.215.125:37333/ using document root <none> and password file <none>
15:07:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.044636 19413 fs_manager.cc:362] Metadata directory not provided
15:07:15.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.044818 19413 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.053017 19413 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.005s
15:07:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.057942 19449 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:15.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.059187 19413 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
15:07:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.059542 19413 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fc71b6722f04408abf55d751c7f83cbb"
15:07:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:49 on dist-test-slave-xht5"
15:07:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.061590 19413 fs_report.cc:389] FS layout report
15:07:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/wal
15:07:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/data
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:15.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.120142 19413 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.128962 19413 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.129582 19413 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.219134 19413 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.125:38877
15:07:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.219178 19500 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.125:38877 every 8 connection(s)
15:07:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.223512 19413 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-1/data/info.pb
15:07:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.226821 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19413
15:07:15.227 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:15.228 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.124:36397
15:07:15.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.229967 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45669
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.243575 19501 sys_catalog.cc:263] Verifying existing consensus state
15:07:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.251961 19501 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap starting.
15:07:15.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.312376 19501 log.cc:826] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Log is configured to *not* fsync() on all Append() calls
15:07:15.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.337044 19501 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:15.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.338147 19501 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Bootstrap complete.
15:07:15.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.377846 19501 raft_consensus.cc:359] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.381651 19501 raft_consensus.cc:740] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: fc71b6722f04408abf55d751c7f83cbb, State: Initialized, Role: FOLLOWER
15:07:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.383423 19501 consensus_queue.cc:260] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.404117 19505 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.405041 19505 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: FOLLOWER
15:07:15.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.407641 19501 sys_catalog.cc:565] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: configured and running, proceeding with master startup.
15:07:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.534094 19516 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:15.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.534375 19516 catalog_manager.cc:1562] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: loading cluster ID for follower catalog manager: success
15:07:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.541725 19516 catalog_manager.cc:1584] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: acquiring CA information for follower catalog manager: success
15:07:15.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.547175 19516 catalog_manager.cc:1612] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:15.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.525636 19413 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:15.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.558946 19413 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.705519 19503 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.706318 19503 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.707580 19503 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:15 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638435, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.724980 18633 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:15.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.759898 18901 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:15.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.764379 19466 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e16d92744730461090a62bf1861a9bd9" instance_seqno: 1765638419369887) as {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:43437; Asking this server to re-register.
15:07:15.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.766340 18901 heartbeater.cc:461] Registering TS with master...
15:07:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.768601 19503 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:15.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.768919 19503 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.769218 19466 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.769402 19503 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.769752 19503 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.770081 19503 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.770445 19503 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.770714 19503 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.817935 19503 master_runner.cc:386] Master server non-default flags:
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.124
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.124:36397
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.124
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45669
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/logs
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.819862 19503 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:15.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.821189 19503 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.832892 19528 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.953915 18751 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.957254 19466 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "95b520c6a290469daf2e1a60c0a06541" instance_seqno: 1765638416377906) as {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:53965; Asking this server to re-register.
15:07:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.958691 18751 heartbeater.cc:461] Registering TS with master...
15:07:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:15.962772 19466 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:07:16.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.053869 19048 heartbeater.cc:344] Connected to a master server at 127.17.215.125:38877
15:07:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.057806 19466 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "fbc5717d67e140648f9ce018c4ce3c15" instance_seqno: 1765638421668181) as {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:44635; Asking this server to re-register.
15:07:16.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.060920 19048 heartbeater.cc:461] Registering TS with master...
15:07:16.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.064929 19466 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:16.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.829295 19542 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:16.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.830466 19542 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:16.865670 19451 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:16.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.871088 19542 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:16.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:16.935571 19451 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.17.215.126:37719, user_credentials={real_user=slave}} blocked reactor thread for 70109.5us
15:07:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:16.938830 19451 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:16.940488 19450 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397): Network error: Client connection negotiation failed: client connection to 127.17.215.124:36397: connect: Connection refused (error 111)
15:07:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.941275 19450 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fc71b6722f04408abf55d751c7f83cbb; no voters: 87ca3bf63db54016822bfee8b1669160, deb58b8805af4e5abe3265e852364b34
15:07:16.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:16.942483 19542 raft_consensus.cc:2749] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
15:07:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:15.834870 19529 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.234691 19527 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 19503
15:07:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.910606 19527 kernel_stack_watchdog.cc:198] Thread 19503 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
15:07:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:07:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:07:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:07:17.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:07:17.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.911607 19503 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.077s user 0.564s sys 1.152s
15:07:17.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.912015 19503 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.078s user 0.564s sys 1.152s
15:07:17.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.912665 19531 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:17.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.916572 19530 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2076 milliseconds
15:07:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.916643 19503 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:07:17.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.918174 19503 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.921233 19503 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:17.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.922648 19503 hybrid_clock.cc:648] HybridClock initialized: now 1765638437922556 us; error 88 us; skew 500 ppm
15:07:17.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.923739 19503 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:17.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:17 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638437, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.931301 19503 init.cc:377] Logged in from keytab as kudu/127.17.215.124@KRBTEST.COM (short username kudu)
15:07:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.937598 19503 webserver.cc:492] Webserver started at http://127.17.215.124:45669/ using document root <none> and password file <none>
15:07:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.938709 19503 fs_manager.cc:362] Metadata directory not provided
15:07:17.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.938952 19503 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:17.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.947269 19503 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
15:07:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.952049 19556 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.953245 19503 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.953589 19503 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87ca3bf63db54016822bfee8b1669160"
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:51 on dist-test-slave-xht5"
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:17.956189 19503 fs_report.cc:389] FS layout report
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/wal
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/data
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:17.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:17.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:17.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.965193 19497 debug-util.cc:398] Leaking SignalData structure 0x7b0800086980 after lost signal to thread 19415
15:07:17.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:17.965999 19497 debug-util.cc:398] Leaking SignalData structure 0x7b0800071540 after lost signal to thread 19500
15:07:18.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.024065 19503 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:18.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.032060 19503 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:18.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.032760 19503 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.154079 19503 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.124:36397
15:07:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.154158 19609 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.124:36397 every 8 connection(s)
15:07:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.158198 19503 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-2/data/info.pb
15:07:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.163843 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19503
15:07:18.164 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:18.164 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.215.126:37719
15:07:18.165 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.176249 18269 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:18.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38781
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:07:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:07:18.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.216846 19610 sys_catalog.cc:263] Verifying existing consensus state
15:07:18.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.229164 19610 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap starting.
15:07:18.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.309983 19610 log.cc:826] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Log is configured to *not* fsync() on all Append() calls
15:07:18.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.342428 19610 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:18.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.343650 19610 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: Bootstrap complete.
15:07:18.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.381567 19610 raft_consensus.cc:359] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:18.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.401161 19610 raft_consensus.cc:740] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 87ca3bf63db54016822bfee8b1669160, State: Initialized, Role: FOLLOWER
15:07:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.402700 19610 consensus_queue.cc:260] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:18.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.422650 19610 sys_catalog.cc:565] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: configured and running, proceeding with master startup.
15:07:18.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.427260 19614 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.430025 19614 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.581815 19625 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:18.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.582119 19625 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: loading cluster ID for follower catalog manager: success
15:07:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.589578 19625 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: acquiring CA information for follower catalog manager: success
15:07:18.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.595059 19625 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:07:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.579612 19503 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:18.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.631057 19503 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:18.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.818559 18902 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:18.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.825279 19574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e16d92744730461090a62bf1861a9bd9" instance_seqno: 1765638419369887) as {username='kudu', principal='kudu/127.17.215.66@KRBTEST.COM'} at 127.17.215.66:47707; Asking this server to re-register.
15:07:18.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.826779 18902 heartbeater.cc:461] Registering TS with master...
15:07:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.830303 19574 ts_manager.cc:194] Registered new tserver with Master: e16d92744730461090a62bf1861a9bd9 (127.17.215.66:43305)
15:07:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:18.915902 19612 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:07:18.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:18.916744 19612 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:07:18.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:18.918037 19612 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:07:18.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.973620 18752 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:18.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.976296 19574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "95b520c6a290469daf2e1a60c0a06541" instance_seqno: 1765638416377906) as {username='kudu', principal='kudu/127.17.215.65@KRBTEST.COM'} at 127.17.215.65:42201; Asking this server to re-register.
15:07:18.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.977847 18752 heartbeater.cc:461] Registering TS with master...
15:07:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:18.980614 19574 ts_manager.cc:194] Registered new tserver with Master: 95b520c6a290469daf2e1a60c0a06541 (127.17.215.65:42129)
15:07:19.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.018540 19612 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:07:19.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.028820 19612 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:07:19.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.029361 19612 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:07:19.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.029762 19612 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:07:19.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.030143 19612 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:07:19.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.033789 19612 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:07:19.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.034103 19612 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40871/jwks.json
15:07:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.044243 19638 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:07:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.051201 19638 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:19.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.060350 19638 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:19.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.081032 19451 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:19 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638439, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.134891 18783 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.140432 19585 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc71b6722f04408abf55d751c7f83cbb" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160" is_pre_election: true
15:07:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.141330 19585 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fc71b6722f04408abf55d751c7f83cbb in term 2.
15:07:19.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.142683 19049 heartbeater.cc:344] Connected to a master server at 127.17.215.124:36397
15:07:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.142566 19450 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, fc71b6722f04408abf55d751c7f83cbb; no voters: deb58b8805af4e5abe3265e852364b34
15:07:19.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.143688 19638 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Leader pre-election won for term 3
15:07:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.144029 19638 raft_consensus.cc:493] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:07:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.144340 19638 raft_consensus.cc:3060] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 2 FOLLOWER]: Advancing to term 3
15:07:19.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.145155 19574 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "fbc5717d67e140648f9ce018c4ce3c15" instance_seqno: 1765638421668181) as {username='kudu', principal='kudu/127.17.215.67@KRBTEST.COM'} at 127.17.215.67:37137; Asking this server to re-register.
15:07:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.146369 19049 heartbeater.cc:461] Registering TS with master...
15:07:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.149175 19574 ts_manager.cc:194] Registered new tserver with Master: fbc5717d67e140648f9ce018c4ce3c15 (127.17.215.67:34603)
15:07:19.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.155462 19638 raft_consensus.cc:515] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.169150 19612 master_runner.cc:386] Master server non-default flags:
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.215.84:42955
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.215.126:37719,127.17.215.125:38877,127.17.215.124:36397
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/krb5kdc/kudu.keytab
15:07:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.215.126
15:07:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:07:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:07:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:07:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.215.126:37719
15:07:19.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40871/jwks.json
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.215.126
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38781
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/logs
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.172290 19585 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "fc71b6722f04408abf55d751c7f83cbb" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "87ca3bf63db54016822bfee8b1669160"
15:07:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.172851 19585 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 2 FOLLOWER]: Advancing to term 3
15:07:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.176811 19451 leader_election.cc:336] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111)
15:07:19.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.179001 19612 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:07:19.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.180291 19638 leader_election.cc:290] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 election: Requested vote from peers deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719), 87ca3bf63db54016822bfee8b1669160 (127.17.215.124:36397)
15:07:19.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.180526 19612 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:07:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.181875 19585 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fc71b6722f04408abf55d751c7f83cbb in term 3.
15:07:19.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.187588 19450 leader_election.cc:304] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 87ca3bf63db54016822bfee8b1669160, fc71b6722f04408abf55d751c7f83cbb; no voters: deb58b8805af4e5abe3265e852364b34
15:07:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.188967 19638 raft_consensus.cc:2804] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 3 FOLLOWER]: Leader election won for term 3
15:07:19.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.206976 19638 raft_consensus.cc:697] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [term 3 LEADER]: Becoming Leader. State: Replica: fc71b6722f04408abf55d751c7f83cbb, State: Running, Role: LEADER
15:07:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.208276 19638 consensus_queue.cc:237] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.226972 19645 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc71b6722f04408abf55d751c7f83cbb. Latest consensus state: current_term: 3 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.227900 19645 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: LEADER
15:07:19.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.238055 19648 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.252058 19649 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:07:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.265486 19649 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:07:19.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.292187 19649 catalog_manager.cc:1269] Loaded cluster ID: 9edd1ea0770144f6b7ef7dcf0b086ab9
15:07:19.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.293316 19649 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:07:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.317345 19649 catalog_manager.cc:1514] Loading token signing keys...
15:07:19.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.325688 19649 catalog_manager.cc:6038] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb: Loaded TSK: 0
15:07:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.338349 19649 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:07:19.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.608626 19585 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [term 3 FOLLOWER]: Refusing update from remote peer fc71b6722f04408abf55d751c7f83cbb: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
15:07:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.611135 19645 consensus_queue.cc:1048] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [LEADER]: Connected to new peer: Peer: permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
15:07:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.648913 19656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: New leader fc71b6722f04408abf55d751c7f83cbb. Latest consensus state: current_term: 3 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.649792 19656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:19.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.677170 19451 consensus_peers.cc:597] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb -> Peer deb58b8805af4e5abe3265e852364b34 (127.17.215.126:37719): Couldn't send request to peer deb58b8805af4e5abe3265e852364b34. Status: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:07:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.678288 19645 sys_catalog.cc:455] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.679199 19645 sys_catalog.cc:458] T 00000000000000000000000000000000 P fc71b6722f04408abf55d751c7f83cbb [sys.catalog]: This master's current role is: LEADER
15:07:19.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.684171 19656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "fc71b6722f04408abf55d751c7f83cbb" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:19.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.684947 19656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 87ca3bf63db54016822bfee8b1669160 [sys.catalog]: This master's current role is: FOLLOWER
15:07:19.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.894522 19044 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae880 after lost signal to thread 18916
15:07:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.895557 19044 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4020 after lost signal to thread 19047
15:07:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:20.478068 18747 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 18612
15:07:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:20.479205 18747 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9fa0 after lost signal to thread 18750
15:07:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:20.626456 19413 master_runner.cc:186] Error getting master registration for 127.17.215.126:37719: Network error: Client connection negotiation failed: client connection to 127.17.215.126:37719: connect: Connection refused (error 111),
15:07:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:20.626919 19413 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:07:20.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:20.791610 18901 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:20.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:20.982259 18751 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:19.263028 19612 server_base.cc:1047] running on GCE node
15:07:21.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.253430 19653 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:19.250936 19650 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:07:21.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.050335 19612 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:07:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.060264 19612 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:07:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.064378 19612 hybrid_clock.cc:648] HybridClock initialized: now 1765638441064342 us; error 47 us; skew 500 ppm
15:07:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.065559 19612 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:07:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:21 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638441, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.073873 19612 init.cc:377] Logged in from keytab as kudu/127.17.215.126@KRBTEST.COM (short username kudu)
15:07:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.083360 19048 heartbeater.cc:499] Master 127.17.215.125:38877 was elected leader, sending a full tablet report...
15:07:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.123201 19612 webserver.cc:492] Webserver started at http://127.17.215.126:38781/ using document root <none> and password file <none>
15:07:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.124533 19612 fs_manager.cc:362] Metadata directory not provided
15:07:21.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.124827 19612 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:07:21.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.148757 19612 fs_manager.cc:714] Time spent opening directory manager: real 0.019s user 0.012s sys 0.008s
15:07:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.162001 19677 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:07:21.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.163388 19612 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.001s sys 0.008s
15:07:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.163807 19612 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "deb58b8805af4e5abe3265e852364b34"
15:07:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:06:47 on dist-test-slave-xht5"
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.167042 19612 fs_report.cc:389] FS layout report
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/wal
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/data
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:07:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:07:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:07:21.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.313890 19612 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:07:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.330850 19612 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:07:21.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.331467 19612 kserver.cc:163] Server-wide thread pool size limit: 3276
15:07:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:07:21 dist-test-slave-xht5 krb5kdc[18281](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638441, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.215.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:07:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.418583 18931 init.cc:295] Successfully reacquired a new kerberos TGT
15:07:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.760990 19612 rpc_server.cc:307] RPC server started. Bound to: 127.17.215.126:37719
15:07:21.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.761080 19728 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.215.126:37719 every 8 connection(s)
15:07:21.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.765184 19612 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster3323565144687777092/master-0/data/info.pb
15:07:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.767956 18269 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 19612
15:07:21.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:07:21.769 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.215.125:38877(127.17.215.125:38877)] connecting to peer
15:07:21.773 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.17.215.125:38877(127.17.215.125:38877)] Successfully connected to peer
15:07:21.798 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.215.125:38877(127.17.215.125:38877)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:07:21.800 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.17.215.125:38877(127.17.215.125:38877)] handling channelInactive
15:07:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.800105 19729 sys_catalog.cc:263] Verifying existing consensus state
15:07:21.804 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.215.126:37719(127.17.215.126:37719)] disconnecting while in state TERMINATED
15:07:21.807 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.215.125:38877(127.17.215.125:38877)] disconnecting while in state TERMINATED
15:07:21.808 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.215.124:36397(127.17.215.124:36397)] disconnecting while in state TERMINATED
15:07:21.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.809849 19729 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap starting.
15:07:21.812 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:21.827 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:21.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:07:21.830569 19731 negotiation.cc:337] Failed RPC negotiation. Trace:
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.773828 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55424 (local address 127.17.215.125:38877)
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.795766 (+ 21938us) server_negotiation.cc:207] Beginning negotiation
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.795796 (+ 30us) server_negotiation.cc:400] Waiting for connection header
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.795856 (+ 60us) server_negotiation.cc:408] Connection header received
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.796018 (+ 162us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.796049 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.796755 (+ 706us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.822231 (+ 25476us) server_negotiation.cc:227] Negotiated authn=SASL
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:07:21.829052 (+ 6821us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55424: BlockingRecv error: recv got EOF from 127.0.0.1:55424 (error 108)
15:07:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21190,"thread_start_us":20875,"threads_started":1}
15:07:21.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.846238 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18610
15:07:21.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.872397 19729 log.cc:826] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Log is configured to *not* fsync() on all Append() calls
15:07:21.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.884580 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18765
15:07:21.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.897684 19729 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:07:21.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.898792 19729 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34: Bootstrap complete.
15:07:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.921397 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18915
15:07:21.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.935891 19729 raft_consensus.cc:359] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:21.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.938434 19729 raft_consensus.cc:740] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: deb58b8805af4e5abe3265e852364b34, State: Initialized, Role: FOLLOWER
15:07:21.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.939445 19729 consensus_queue.cc:260] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } }
15:07:21.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.942683 19737 sys_catalog.cc:455] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "deb58b8805af4e5abe3265e852364b34" member_type: VOTER last_known_addr { host: "127.17.215.126" port: 37719 } } peers { permanent_uuid: "fc71b6722f04408abf55d751c7f83cbb" member_type: VOTER last_known_addr { host: "127.17.215.125" port: 38877 } } peers { permanent_uuid: "87ca3bf63db54016822bfee8b1669160" member_type: VOTER last_known_addr { host: "127.17.215.124" port: 36397 } } }
15:07:21.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.943622 19737 sys_catalog.cc:458] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: This master's current role is: FOLLOWER
15:07:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.944679 19729 sys_catalog.cc:565] T 00000000000000000000000000000000 P deb58b8805af4e5abe3265e852364b34 [sys.catalog]: configured and running, proceeding with master startup.
15:07:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.963411 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19612
15:07:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:21.985507 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19413
15:07:22.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:22.024230 18269 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 19503
15:07:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:07:23Z chronyd exiting
15:07:24.266 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:07:24.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.268059 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 17828
15:07:24.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.293282 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 17986
15:07:24.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.320417 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 18127
15:07:24.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.345679 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 17556
15:07:24.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.377650 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 17626
15:07:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:07:24.407720 17544 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 17701
15:07:25.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:07:25Z chronyd exiting
15:07:26.584 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more