14:10:23.754 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:10:23.754 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLIfY9B/test-tmp
14:10:23.754 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:10:23.754 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLIfY9B/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:10:23.756 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:10:24.065 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:10:24.065 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:10:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.068307 32275 mini_oidc.cc:150] Starting JWKS server
14:10:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.071660 32275 webserver.cc:469] Webserver started at http://127.0.0.1:44345/ using document root <none> and password file <none>
14:10:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.072129 32275 mini_oidc.cc:186] Starting OIDC Discovery server
14:10:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.073763 32275 webserver.cc:469] Webserver started at http://127.0.0.1:46769/ using document root <none> and password file <none>
14:10:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:10:24Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:10:24Z Disabled control of system clock
14:10:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.132573 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:24.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/logs
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data/info.pb
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.254:37737
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.254
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:24.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json with env {}
14:10:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.412000 32287 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.412537 32287 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:24.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.413000 32287 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.442505 32287 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:24.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.442786 32287 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.442982 32287 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.443167 32287 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.443344 32287 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:24.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.443477 32287 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44345/jwks.json
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.476004 32287 master_runner.cc:386] Master server non-default flags:
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.254:37737
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data/info.pb
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.254
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/logs
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:24.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.477200 32287 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:24.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.478612 32287 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.493090 32293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.493100 32294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:24.493145 32296 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:25.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:24.495013 32287 server_base.cc:1048] running on GCE node
14:10:25.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.609850 32287 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.612845 32287 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:25.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.614289 32287 hybrid_clock.cc:648] HybridClock initialized: now 1751033425614257 us; error 65 us; skew 500 ppm
14:10:25.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.615008 32287 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.628788 32287 webserver.cc:469] Webserver started at http://127.31.132.254:34803/ using document root <none> and password file <none>
14:10:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.629652 32287 fs_manager.cc:362] Metadata directory not provided
14:10:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.629837 32287 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.630234 32287 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:25.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.634368 32287 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data/instance:
14:10:25.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "438d47d532254abe81475ea910e95cf3"
14:10:25.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:25 on dist-test-slave-pbh5"
14:10:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.635380 32287 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal/instance:
14:10:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "438d47d532254abe81475ea910e95cf3"
14:10:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:25 on dist-test-slave-pbh5"
14:10:25.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.642081 32287 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
14:10:25.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.647207 32303 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.648123 32287 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
14:10:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.648383 32287 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal
14:10:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "438d47d532254abe81475ea910e95cf3"
14:10:25.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:25 on dist-test-slave-pbh5"
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.648692 32287 fs_report.cc:389] FS layout report
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data/data
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:25.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:25.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.712018 32287 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:25.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.713456 32287 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:25.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.713809 32287 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.779553 32287 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.254:37737
14:10:25.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.779659 32354 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.254:37737 every 8 connection(s)
14:10:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.782217 32287 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/data/info.pb
14:10:25.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.786036 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 32287
14:10:25.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.786540 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-0/wal/instance
14:10:25.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.787819 32355 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.791708 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/logs
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data/info.pb
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.253:34875
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.253
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:25.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:25.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json with env {}
14:10:25.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.802021 32355 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } has no permanent_uuid. Determining permanent_uuid...
14:10:25.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.826257 32355 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } has no permanent_uuid. Determining permanent_uuid...
14:10:25.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:25.828452 32307 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:25.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:25.832377 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:25.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.881836 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 1
14:10:25.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:25.886040 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:25.954423 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 2
14:10:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:25.958520 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:26.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.049875 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 3
14:10:26.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.053651 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.086968 32357 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:26.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.087534 32357 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:26.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.088053 32357 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.117096 32357 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.117411 32357 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:26.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.117689 32357 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.117944 32357 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.118218 32357 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.118420 32357 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44345/jwks.json
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.150791 32357 master_runner.cc:386] Master server non-default flags:
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.253:34875
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data/info.pb
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.253
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/logs
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:26.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:26.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.152315 32357 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:26.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.153934 32357 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:26.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.167047 32367 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:26.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.196974 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 4
14:10:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.200372 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:26.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:26.499974 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 5
14:10:26.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.503857 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.051424 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 6
14:10:27.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.058557 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.253:34875: Network error: Client connection negotiation failed: client connection to 127.31.132.253:34875: connect: Connection refused (error 111)
14:10:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.167312 32369 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:26.169343 32366 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.310057 32368 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:10:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.310180 32357 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:27.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.314841 32357 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.317976 32357 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.319444 32357 hybrid_clock.cc:648] HybridClock initialized: now 1751033427319375 us; error 77 us; skew 500 ppm
14:10:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.320447 32357 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:27.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.327069 32357 webserver.cc:469] Webserver started at http://127.31.132.253:45243/ using document root <none> and password file <none>
14:10:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.327975 32357 fs_manager.cc:362] Metadata directory not provided
14:10:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.328168 32357 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:27.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.328627 32357 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.333031 32357 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data/instance:
14:10:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db4ddaae82449edae3320a58163035b"
14:10:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:27 on dist-test-slave-pbh5"
14:10:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.334036 32357 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal/instance:
14:10:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db4ddaae82449edae3320a58163035b"
14:10:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:27 on dist-test-slave-pbh5"
14:10:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.340767 32357 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.003s sys 0.004s
14:10:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.346060 32378 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.347090 32357 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:10:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.347358 32357 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal
14:10:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0db4ddaae82449edae3320a58163035b"
14:10:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:27 on dist-test-slave-pbh5"
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.347638 32357 fs_report.cc:389] FS layout report
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data/data
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:27.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.419356 32357 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.420764 32357 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.421139 32357 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.489862 32357 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.253:34875
14:10:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.489974 32429 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.253:34875 every 8 connection(s)
14:10:27.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.492419 32357 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/data/info.pb
14:10:27.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.497215 32430 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.500883 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 32357
14:10:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.501202 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-1/wal/instance
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.502979 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/logs
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data/info.pb
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.252:41659
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.252
14:10:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json with env {}
14:10:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.508522 32430 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } has no permanent_uuid. Determining permanent_uuid...
14:10:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.532313 32430 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } has no permanent_uuid. Determining permanent_uuid...
14:10:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.545126 32430 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } has no permanent_uuid. Determining permanent_uuid...
14:10:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.547209 32381 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.550881 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:27.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.600320 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 1
14:10:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.604130 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:27.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.672470 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 2
14:10:27.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.676096 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:27.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.767427 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 3
14:10:27.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.770995 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.795046 32432 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:27.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.795552 32432 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:27.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.796000 32432 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.825917 32432 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.826159 32432 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.826335 32432 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.826534 32432 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.826722 32432 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:27.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.826859 32432 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44345/jwks.json
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.859825 32432 master_runner.cc:386] Master server non-default flags:
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.252:41659
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44345/jwks.json
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data/info.pb
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.252
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/logs
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:27.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:27.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.861074 32432 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:27.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.862617 32432 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:27.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.877385 32443 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:27.914427 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 4
14:10:27.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.920452 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.119122 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } attempt: 7
14:10:28.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.148742 32355 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } has no permanent_uuid. Determining permanent_uuid...
14:10:28.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.156531 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.215041 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 1
14:10:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.219964 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 5
14:10:28.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.225898 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.226408 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.307651 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 2
14:10:28.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.313863 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.399397 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 3
14:10:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.405539 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.546072 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 4
14:10:28.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.552601 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.773905 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 6
14:10:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.791653 32430 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:28.836225 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 5
14:10:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:28.842566 32355 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.31.132.252:41659: Network error: Client connection negotiation failed: client connection to 127.31.132.252:41659: connect: Connection refused (error 111)
14:10:29.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.877506 32445 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:29.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:27.877494 32442 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:29.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.087641 32444 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1208 milliseconds
14:10:29.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.087769 32432 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:29.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.089043 32432 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:29.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.091473 32432 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.092829 32432 hybrid_clock.cc:648] HybridClock initialized: now 1751033429092801 us; error 37 us; skew 500 ppm
14:10:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.093605 32432 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:29.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.099666 32432 webserver.cc:469] Webserver started at http://127.31.132.252:38539/ using document root <none> and password file <none>
14:10:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.100502 32432 fs_manager.cc:362] Metadata directory not provided
14:10:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.100705 32432 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:29.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.101112 32432 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.105399 32432 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data/instance:
14:10:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e980a9b2cc854986aed2704178355d76"
14:10:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:29 on dist-test-slave-pbh5"
14:10:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.106478 32432 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal/instance:
14:10:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e980a9b2cc854986aed2704178355d76"
14:10:29.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:29 on dist-test-slave-pbh5"
14:10:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.113263 32432 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
14:10:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.118352 32457 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.119277 32432 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.004s sys 0.001s
14:10:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.119563 32432 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal
14:10:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e980a9b2cc854986aed2704178355d76"
14:10:29.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:29 on dist-test-slave-pbh5"
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.119830 32432 fs_report.cc:389] FS layout report
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data/data
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:29.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:29.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.185782 32432 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.187178 32432 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:29.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.187561 32432 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:29.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.258222 32432 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.252:41659
14:10:29.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.258281 32508 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.252:41659 every 8 connection(s)
14:10:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.260766 32432 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/data/info.pb
14:10:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.265602 32509 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.270960 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 32432
14:10:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.271462 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/master-2/wal/instance
14:10:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.279213 32509 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } has no permanent_uuid. Determining permanent_uuid...
14:10:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.303701 32509 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } has no permanent_uuid. Determining permanent_uuid...
14:10:29.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.316702 32509 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } has no permanent_uuid. Determining permanent_uuid...
14:10:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.331176 32509 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: Bootstrap starting.
14:10:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.336558 32509 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: Neither blocks nor log segments found. Creating new log.
14:10:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.338552 32509 log.cc:826] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: Log is configured to *not* fsync() on all Append() calls
14:10:29.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.342494 32509 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: No bootstrap required, opened a new log
14:10:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.359639 32509 raft_consensus.cc:357] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.360227 32509 raft_consensus.cc:383] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.360417 32509 raft_consensus.cc:738] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e980a9b2cc854986aed2704178355d76, State: Initialized, Role: FOLLOWER
14:10:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.361121 32509 consensus_queue.cc:260] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.363214 32519 sys_catalog.cc:455] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.363978 32519 sys_catalog.cc:458] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: This master's current role is: FOLLOWER
14:10:29.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.364674 32509 sys_catalog.cc:564] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: configured and running, proceeding with master startup.
14:10:29.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.389195 32530 catalog_manager.cc:1560] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:29.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.389509 32530 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:29.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.395148 32355 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 6
14:10:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.384857 32432 master_runner.cc:186] Error getting master registration for 127.31.132.254:37737: OK, instance_id { permanent_uuid: "438d47d532254abe81475ea910e95cf3" instance_seqno: 1751033425761713 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.410403 32432 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.414407 32355 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: Bootstrap starting.
14:10:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.419421 32355 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: Neither blocks nor log segments found. Creating new log.
14:10:29.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.421077 32355 log.cc:826] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: Log is configured to *not* fsync() on all Append() calls
14:10:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.425562 32355 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: No bootstrap required, opened a new log
14:10:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.442368 32463 tablet.cc:2378] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.442721 32355 raft_consensus.cc:357] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.443548 32355 raft_consensus.cc:383] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.443773 32355 raft_consensus.cc:738] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 438d47d532254abe81475ea910e95cf3, State: Initialized, Role: FOLLOWER
14:10:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.444721 32355 consensus_queue.cc:260] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.447793 32534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.448647 32534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.449606 32355 sys_catalog.cc:564] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: configured and running, proceeding with master startup.
14:10:29.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.472522 32309 tablet.cc:2378] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.481599 32545 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:29.481994 32545 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.478432 32287 master_runner.cc:186] Error getting master registration for 127.31.132.253:34875: OK, instance_id { permanent_uuid: "0db4ddaae82449edae3320a58163035b" instance_seqno: 1751033427470777 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:29.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.507561 32287 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.628875 32519 raft_consensus.cc:491] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:29.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.629415 32519 raft_consensus.cc:513] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.631788 32519 leader_election.cc:290] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 438d47d532254abe81475ea910e95cf3 (127.31.132.254:37737), 0db4ddaae82449edae3320a58163035b (127.31.132.253:34875)
14:10:29.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.632948 32330 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e980a9b2cc854986aed2704178355d76" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "438d47d532254abe81475ea910e95cf3" is_pre_election: true
14:10:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.633795 32330 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e980a9b2cc854986aed2704178355d76 in term 0.
14:10:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.635044 32459 leader_election.cc:304] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 438d47d532254abe81475ea910e95cf3, e980a9b2cc854986aed2704178355d76; no voters:
14:10:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.635845 32519 raft_consensus.cc:2802] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:10:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.636166 32519 raft_consensus.cc:491] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.636471 32519 raft_consensus.cc:3058] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 0 FOLLOWER]: Advancing to term 1
14:10:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.643412 32519 raft_consensus.cc:513] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.645931 32330 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e980a9b2cc854986aed2704178355d76" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "438d47d532254abe81475ea910e95cf3"
14:10:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.646468 32330 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 0 FOLLOWER]: Advancing to term 1
14:10:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.646411 32519 leader_election.cc:290] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [CANDIDATE]: Term 1 election: Requested vote from peers 438d47d532254abe81475ea910e95cf3 (127.31.132.254:37737), 0db4ddaae82449edae3320a58163035b (127.31.132.253:34875)
14:10:29.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.659468 32330 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e980a9b2cc854986aed2704178355d76 in term 1.
14:10:29.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.669561 32459 leader_election.cc:304] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 438d47d532254abe81475ea910e95cf3, e980a9b2cc854986aed2704178355d76; no voters:
14:10:29.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.670369 32519 raft_consensus.cc:2802] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 1 FOLLOWER]: Leader election won for term 1
14:10:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.678560 32519 raft_consensus.cc:695] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [term 1 LEADER]: Becoming Leader. State: Replica: e980a9b2cc854986aed2704178355d76, State: Running, Role: LEADER
14:10:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.679474 32519 consensus_queue.cc:237] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.691176 32547 sys_catalog.cc:455] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e980a9b2cc854986aed2704178355d76. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.692030 32547 sys_catalog.cc:458] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: This master's current role is: LEADER
14:10:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.698637 32549 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.705767 32549 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.731015 32330 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [term 1 FOLLOWER]: Refusing update from remote peer e980a9b2cc854986aed2704178355d76: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.732731 32519 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [LEADER]: Connected to new peer: Peer: permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.757860 32534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e980a9b2cc854986aed2704178355d76. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.758538 32534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:29.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.777086 32550 mvcc.cc:204] Tried to move back new op lower bound from 7172232928132202496 to 7172232927977230336. Current Snapshot: MvccSnapshot[applied={T|T < 7172232928132202496}]
14:10:29.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.787896 32551 mvcc.cc:204] Tried to move back new op lower bound from 7172232928132202496 to 7172232927977230336. Current Snapshot: MvccSnapshot[applied={T|T < 7172232928132202496}]
14:10:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.789800 32547 sys_catalog.cc:455] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.790536 32547 sys_catalog.cc:458] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: This master's current role is: LEADER
14:10:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.796989 32549 catalog_manager.cc:1349] Generated new cluster ID: a12b7bcfc0474018a13f9570037cae6b
14:10:29.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.797271 32549 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:29.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.827821 32534 sys_catalog.cc:455] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:29.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.828464 32534 sys_catalog.cc:458] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3 [sys.catalog]: This master's current role is: FOLLOWER
14:10:29.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.841074 32549 catalog_manager.cc:1372] Generated new certificate authority record
14:10:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.842988 32549 catalog_manager.cc:1506] Loading token signing keys...
14:10:29.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.852142 32430 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } attempt: 7
14:10:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.874210 32549 catalog_manager.cc:5955] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76: Generated new TSK 0
14:10:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.875293 32549 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.896811 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/logs
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data/info.pb
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.193:0
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.193
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.193
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:29.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.924840 32430 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: Bootstrap starting.
14:10:29.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.931937 32430 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: Neither blocks nor log segments found. Creating new log.
14:10:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.935276 32430 log.cc:826] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: Log is configured to *not* fsync() on all Append() calls
14:10:29.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:29.953914 32430 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: No bootstrap required, opened a new log
14:10:30.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.018630 32430 raft_consensus.cc:357] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:30.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.020956 32430 raft_consensus.cc:383] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.022279 32430 raft_consensus.cc:738] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0db4ddaae82449edae3320a58163035b, State: Initialized, Role: FOLLOWER
14:10:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.024735 32430 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } }
14:10:30.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.072029 32558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.073122 32558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: This master's current role is: FOLLOWER
14:10:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.073784 32430 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: configured and running, proceeding with master startup.
14:10:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.074286 32405 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e980a9b2cc854986aed2704178355d76" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0db4ddaae82449edae3320a58163035b" is_pre_election: true
14:10:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.075284 32403 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [term 0 FOLLOWER]: Advancing to term 1
14:10:30.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.075726 32404 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e980a9b2cc854986aed2704178355d76" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0db4ddaae82449edae3320a58163035b"
14:10:30.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.087198 32403 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [term 1 FOLLOWER]: Refusing update from remote peer e980a9b2cc854986aed2704178355d76: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:30.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.091073 32519 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:30.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.153087 32558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: SysCatalogTable state changed. Reason: New leader e980a9b2cc854986aed2704178355d76. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:30.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.153832 32558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: This master's current role is: FOLLOWER
14:10:30.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.154942 32567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.155797 32567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b [sys.catalog]: This master's current role is: FOLLOWER
14:10:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.173177 32574 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.392990 32556 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:30.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.393610 32556 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.394357 32556 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.450558 32556 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.451932 32556 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.132.193
14:10:30.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.493597 32545 catalog_manager.cc:1261] Loaded cluster ID: a12b7bcfc0474018a13f9570037cae6b
14:10:30.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.495777 32545 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: loading cluster ID for follower catalog manager: success
14:10:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.509927 32545 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: acquiring CA information for follower catalog manager: success
14:10:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.512599 32556 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:30.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.193:0
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data/info.pb
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.193
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.193
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/logs
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:30.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.514256 32556 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:30.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.516341 32556 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.524123 32545 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 438d47d532254abe81475ea910e95cf3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:30.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.560317 32581 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:30.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.658973 32519 sys_catalog.cc:455] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e980a9b2cc854986aed2704178355d76" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "438d47d532254abe81475ea910e95cf3" member_type: VOTER last_known_addr { host: "127.31.132.254" port: 37737 } } peers { permanent_uuid: "0db4ddaae82449edae3320a58163035b" member_type: VOTER last_known_addr { host: "127.31.132.253" port: 34875 } } peers { permanent_uuid: "e980a9b2cc854986aed2704178355d76" member_type: VOTER last_known_addr { host: "127.31.132.252" port: 41659 } } }
14:10:30.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.659992 32519 sys_catalog.cc:458] T 00000000000000000000000000000000 P e980a9b2cc854986aed2704178355d76 [sys.catalog]: This master's current role is: LEADER
14:10:31.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.193421 32574 catalog_manager.cc:1261] Loaded cluster ID: a12b7bcfc0474018a13f9570037cae6b
14:10:31.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.196063 32574 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: loading cluster ID for follower catalog manager: success
14:10:31.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.207897 32574 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: acquiring CA information for follower catalog manager: success
14:10:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.223699 32574 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 0db4ddaae82449edae3320a58163035b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:30.562496 32556 server_base.cc:1048] running on GCE node
14:10:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.568928 32580 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:30.564998 32584 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.930346 32556 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.932966 32556 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:31.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.934455 32556 hybrid_clock.cc:648] HybridClock initialized: now 1751033431934421 us; error 44 us; skew 500 ppm
14:10:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.935479 32556 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.960152 32556 webserver.cc:469] Webserver started at http://127.31.132.193:33345/ using document root <none> and password file <none>
14:10:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.961468 32556 fs_manager.cc:362] Metadata directory not provided
14:10:31.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.961741 32556 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:31.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.962316 32556 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.968585 32556 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data/instance:
14:10:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54342b49289842c889bc93450773ee80"
14:10:31.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:31 on dist-test-slave-pbh5"
14:10:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.970000 32556 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal/instance:
14:10:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54342b49289842c889bc93450773ee80"
14:10:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:31 on dist-test-slave-pbh5"
14:10:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.978832 32556 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
14:10:31.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.985682 32594 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.986824 32556 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:10:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.987190 32556 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal
14:10:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54342b49289842c889bc93450773ee80"
14:10:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:31 on dist-test-slave-pbh5"
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:31.987574 32556 fs_report.cc:389] FS layout report
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data/data
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.061761 32556 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:32.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.063522 32556 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:32.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.063994 32556 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:32.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.067052 32556 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.072243 32556 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.072490 32556 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:10:32.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.072752 32556 ts_tablet_manager.cc:610] Registered 0 tablets
14:10:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.072932 32556 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.268971 32556 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.193:35659
14:10:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.269173 32706 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.193:35659 every 8 connection(s)
14:10:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.272446 32556 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/data/info.pb
14:10:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.275089 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 32556
14:10:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.275537 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-0/wal/instance
14:10:32.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.289599 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/logs
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data/info.pb
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.194:0
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.194
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.194
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:32.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.344584 32714 heartbeater.cc:344] Connected to a master server at 127.31.132.254:37737
14:10:32.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.345100 32714 heartbeater.cc:461] Registering TS with master...
14:10:32.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.346393 32714 heartbeater.cc:507] Master 127.31.132.254:37737 requested a full tablet report, sending...
14:10:32.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.347594 32707 heartbeater.cc:344] Connected to a master server at 127.31.132.253:34875
14:10:32.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.347966 32707 heartbeater.cc:461] Registering TS with master...
14:10:32.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.348091 32710 heartbeater.cc:344] Connected to a master server at 127.31.132.252:41659
14:10:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.348387 32710 heartbeater.cc:461] Registering TS with master...
14:10:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.348825 32707 heartbeater.cc:507] Master 127.31.132.253:34875 requested a full tablet report, sending...
14:10:32.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.349216 32710 heartbeater.cc:507] Master 127.31.132.252:41659 requested a full tablet report, sending...
14:10:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.351405 32474 ts_manager.cc:194] Registered new tserver with Master: 54342b49289842c889bc93450773ee80 (127.31.132.193:35659)
14:10:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.351712 32395 ts_manager.cc:194] Registered new tserver with Master: 54342b49289842c889bc93450773ee80 (127.31.132.193:35659)
14:10:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.352196 32320 ts_manager.cc:194] Registered new tserver with Master: 54342b49289842c889bc93450773ee80 (127.31.132.193:35659)
14:10:32.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.353828 32474 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.132.193:60611
14:10:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.663048 32711 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:32.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.663555 32711 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:32.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.664089 32711 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.694402 32711 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.695266 32711 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.132.194
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.728009 32711 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.194:0
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data/info.pb
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.194
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.194
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/logs
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.729327 32711 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:32.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.730757 32711 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.747511 32724 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:33.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.356689 32710 heartbeater.cc:499] Master 127.31.132.252:41659 was elected leader, sending a full tablet report...
14:10:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.748396 32723 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:32.750823 32711 server_base.cc:1048] running on GCE node
14:10:33.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:32.748817 32726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.882864 32711 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:33.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.885494 32711 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.886917 32711 hybrid_clock.cc:648] HybridClock initialized: now 1751033433886871 us; error 63 us; skew 500 ppm
14:10:33.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.887704 32711 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.893488 32711 webserver.cc:469] Webserver started at http://127.31.132.194:44699/ using document root <none> and password file <none>
14:10:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.894333 32711 fs_manager.cc:362] Metadata directory not provided
14:10:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.894512 32711 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:33.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.894903 32711 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.899012 32711 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data/instance:
14:10:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac069e16b8e546d3aea19e760f32475a"
14:10:33.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:33 on dist-test-slave-pbh5"
14:10:33.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.899920 32711 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal/instance:
14:10:33.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac069e16b8e546d3aea19e760f32475a"
14:10:33.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:33 on dist-test-slave-pbh5"
14:10:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.906347 32711 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.004s
14:10:33.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.911500 32733 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:33.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.912353 32711 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.912700 32711 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac069e16b8e546d3aea19e760f32475a"
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:33 on dist-test-slave-pbh5"
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.913008 32711 fs_report.cc:389] FS layout report
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data/data
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:33.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:33.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:33.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.968245 32711 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:33.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.969619 32711 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:33.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.970081 32711 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.972579 32711 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.976536 32711 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.976756 32711 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:10:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.977010 32711 ts_tablet_manager.cc:610] Registered 0 tablets
14:10:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:33.977144 32711 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.116796 32711 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.194:35485
14:10:34.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.116895 378 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.194:35485 every 8 connection(s)
14:10:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.119258 32711 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/data/info.pb
14:10:34.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.122714 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 32711
14:10:34.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.123289 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-1/wal/instance
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.138934 32275 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/logs
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data/info.pb
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.195:0
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.195
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.195
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:34.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.183017 379 heartbeater.cc:344] Connected to a master server at 127.31.132.253:34875
14:10:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.183460 379 heartbeater.cc:461] Registering TS with master...
14:10:34.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.184607 379 heartbeater.cc:507] Master 127.31.132.253:34875 requested a full tablet report, sending...
14:10:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.185734 380 heartbeater.cc:344] Connected to a master server at 127.31.132.252:41659
14:10:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.186105 380 heartbeater.cc:461] Registering TS with master...
14:10:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.186375 383 heartbeater.cc:344] Connected to a master server at 127.31.132.254:37737
14:10:34.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.186659 383 heartbeater.cc:461] Registering TS with master...
14:10:34.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.187021 380 heartbeater.cc:507] Master 127.31.132.252:41659 requested a full tablet report, sending...
14:10:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.187453 32395 ts_manager.cc:194] Registered new tserver with Master: ac069e16b8e546d3aea19e760f32475a (127.31.132.194:35485)
14:10:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.187515 383 heartbeater.cc:507] Master 127.31.132.254:37737 requested a full tablet report, sending...
14:10:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.189169 32474 ts_manager.cc:194] Registered new tserver with Master: ac069e16b8e546d3aea19e760f32475a (127.31.132.194:35485)
14:10:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.190379 32320 ts_manager.cc:194] Registered new tserver with Master: ac069e16b8e546d3aea19e760f32475a (127.31.132.194:35485)
14:10:34.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.190727 32474 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.132.194:52997
14:10:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.470861 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.471294 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:34.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.471726 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.502573 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.503329 386 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.132.195
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.540791 386 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.132.212:45335
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.132.195:0
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data/info.pb
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.132.195
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.132.254:37737,127.31.132.253:34875,127.31.132.252:41659
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.132.195
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/logs
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:34.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.542100 386 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:34.543661 386 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:34.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.558176 396 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:35.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.193367 380 heartbeater.cc:499] Master 127.31.132.252:41659 was elected leader, sending a full tablet report...
14:10:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:34.558457 397 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:35.749783 399 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:35.752200 398 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1191 milliseconds
14:10:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.752337 386 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.753563 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:35.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.756107 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:35.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.757529 386 hybrid_clock.cc:648] HybridClock initialized: now 1751033435757474 us; error 74 us; skew 500 ppm
14:10:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.758245 386 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.764412 386 webserver.cc:469] Webserver started at http://127.31.132.195:45701/ using document root <none> and password file <none>
14:10:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.765319 386 fs_manager.cc:362] Metadata directory not provided
14:10:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.765529 386 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:35.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.765995 386 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.770313 386 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data/instance:
14:10:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a6a40dfe9674a54942069a4e5511407"
14:10:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:35 on dist-test-slave-pbh5"
14:10:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.771361 386 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal/instance:
14:10:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a6a40dfe9674a54942069a4e5511407"
14:10:35.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:35 on dist-test-slave-pbh5"
14:10:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.778187 386 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
14:10:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.784566 407 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.785616 386 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.785920 386 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a6a40dfe9674a54942069a4e5511407"
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:35 on dist-test-slave-pbh5"
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.786237 386 fs_report.cc:389] FS layout report
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data/data
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:35.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:35.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.837579 386 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.839039 386 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:35.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.839486 386 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:35.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.841943 386 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.845956 386 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.846138 386 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:10:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.846319 386 ts_tablet_manager.cc:610] Registered 0 tablets
14:10:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.846443 386 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.980409 386 rpc_server.cc:307] RPC server started. Bound to: 127.31.132.195:44899
14:10:35.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.980487 519 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.132.195:44899 every 8 connection(s)
14:10:35.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.982918 386 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/data/info.pb
14:10:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.990805 32275 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 386
14:10:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:35.991364 32275 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster8515912207348256872/ts-2/wal/instance
14:10:36.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.059623 521 heartbeater.cc:344] Connected to a master server at 127.31.132.252:41659
14:10:36.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.060176 521 heartbeater.cc:461] Registering TS with master...
14:10:36.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.061486 521 heartbeater.cc:507] Master 127.31.132.252:41659 requested a full tablet report, sending...
14:10:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.064031 32474 ts_manager.cc:194] Registered new tserver with Master: 6a6a40dfe9674a54942069a4e5511407 (127.31.132.195:44899)
14:10:36.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.065660 32474 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.132.195:60721
14:10:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.068164 520 heartbeater.cc:344] Connected to a master server at 127.31.132.253:34875
14:10:36.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.068538 520 heartbeater.cc:461] Registering TS with master...
14:10:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.069239 520 heartbeater.cc:507] Master 127.31.132.253:34875 requested a full tablet report, sending...
14:10:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.070292 522 heartbeater.cc:344] Connected to a master server at 127.31.132.254:37737
14:10:36.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.070631 522 heartbeater.cc:461] Registering TS with master...
14:10:36.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.071211 522 heartbeater.cc:507] Master 127.31.132.254:37737 requested a full tablet report, sending...
14:10:36.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.073421 32320 ts_manager.cc:194] Registered new tserver with Master: 6a6a40dfe9674a54942069a4e5511407 (127.31.132.195:44899)
14:10:36.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.085649 32395 ts_manager.cc:194] Registered new tserver with Master: 6a6a40dfe9674a54942069a4e5511407 (127.31.132.195:44899)
14:10:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.091527 32275 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
14:10:36.102 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:10:36.102 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:10:36.103 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.31.132.254"
port: 37737
}
bound_http_address {
host: "127.31.132.254"
port: 34803
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.31.132.253"
port: 34875
}
bound_http_address {
host: "127.31.132.253"
port: 45243
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.31.132.252"
port: 41659
}
bound_http_address {
host: "127.31.132.252"
port: 38539
}
}
}
14:10:36.103 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:10:36.104 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.31.132.193"
port: 35659
}
bound_http_address {
host: "127.31.132.193"
port: 33345
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.31.132.194"
port: 35485
}
bound_http_address {
host: "127.31.132.194"
port: 44699
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.31.132.195"
port: 44899
}
bound_http_address {
host: "127.31.132.195"
port: 45701
}
}
}
14:10:36.104 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:10:36.104 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster5307319206234385441/krb5kdc/krb5cc
14:10:36.105 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:10:36.106 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLIfY9B/test-tmp
14:10:36.106 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:10:36.106 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLIfY9B/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:10:36.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:10:36.437 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:36.437 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:10:36.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.441128 532 mini_oidc.cc:150] Starting JWKS server
14:10:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.444167 532 webserver.cc:469] Webserver started at http://127.0.0.1:34571/ using document root <none> and password file <none>
14:10:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.444633 532 mini_oidc.cc:186] Starting OIDC Discovery server
14:10:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:36.446252 532 webserver.cc:469] Webserver started at http://127.0.0.1:44775/ using document root <none> and password file <none>
14:10:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:10:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/principal' for realm 'KRBTEST.COM',
14:10:36.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:10:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:36 dist-test-slave-pbh5 krb5kdc[543](info): setting up network...
14:10:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:10:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:36 dist-test-slave-pbh5 krb5kdc[543](info): set up 2 sockets
14:10:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:36 dist-test-slave-pbh5 krb5kdc[543](info): commencing operation
14:10:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:10:37.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:37.070197 521 heartbeater.cc:499] Master 127.31.132.252:41659 was elected leader, sending a full tablet report...
14:10:38.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.478013 532 mini_kdc.cc:121] Time spent starting KDC: real 1.996s user 0.000s sys 0.016s
14:10:38.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:10:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:38.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:10:38.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:10:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:38.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:10:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:10:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:38.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:10:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:10:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/test-user.keytab.
14:10:38.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/test-user.keytab.
14:10:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:38 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033438, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:10:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:10:38Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:38.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:10:38Z Disabled control of system clock
14:10:38.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.133.62@KRBTEST.COM; defaulting to no policy
14:10:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:38.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.133.62@KRBTEST.COM" created.
14:10:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab.
14:10:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab.
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:38.657467 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.934633 559 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.935153 559 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:38.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.935712 559 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.965267 559 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.965487 559 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:38.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.965759 559 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.965947 559 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.966131 559 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.966391 559 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:38.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:38.966558 559 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:39.003975 559 master_runner.cc:386] Master server non-default flags:
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:39.005318 559 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:39.006361 559 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:39.022147 566 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:39.022614 568 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:39.025193 559 server_base.cc:1048] running on GCE node
14:10:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:39.023233 565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:40.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.209741 559 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:40.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.213263 559 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:40.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.214723 559 hybrid_clock.cc:648] HybridClock initialized: now 1751033440214644 us; error 78 us; skew 500 ppm
14:10:40.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.215500 559 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:40.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:40 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033440, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:40.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.221109 559 init.cc:377] Logged in from keytab as kudu/127.0.133.62@KRBTEST.COM (short username kudu)
14:10:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.225991 559 webserver.cc:469] Webserver started at http://127.0.133.62:38243/ using document root <none> and password file <none>
14:10:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.226784 559 fs_manager.cc:362] Metadata directory not provided
14:10:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.226969 559 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:40.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.227330 559 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.231511 559 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/instance:
14:10:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:10:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.232591 559 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal/instance:
14:10:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:40.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:10:40.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.239182 559 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.002s sys 0.004s
14:10:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.244372 576 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:40.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.245494 559 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.245801 559 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.246114 559 fs_report.cc:389] FS layout report
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/data
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.293740 559 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.298961 559 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.299412 559 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.374778 559 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.62:34635
14:10:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.374800 628 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.62:34635 every 8 connection(s)
14:10:40.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.377621 559 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.383169 629 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.384059 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 559
14:10:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.384509 532 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal/instance
14:10:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.398463 629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
14:10:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.133.61@KRBTEST.COM; defaulting to no policy
14:10:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.133.61@KRBTEST.COM" created.
14:10:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:40 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033440, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab.
14:10:40.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab.
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.432106 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:40.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.435084 629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } has no permanent_uuid. Determining permanent_uuid...
14:10:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.437469 577 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.441015 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.490422 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 1
14:10:40.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.493718 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.562057 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 2
14:10:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.565611 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.657029 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 3
14:10:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.661192 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.729336 635 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.729820 635 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:40.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.730329 635 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.759599 635 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.759845 635 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.760201 635 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.760408 635 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.760612 635 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.760798 635 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:40.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.760936 635 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.804558 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 4
14:10:40.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.807978 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.810686 635 master_runner.cc:386] Master server non-default flags:
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:40.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.812286 635 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:40.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.813560 635 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:40.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.829110 643 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:41.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:41.107293 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 5
14:10:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:41.111447 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:41.658794 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 6
14:10:41.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:41.665308 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:42.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.829176 642 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:42.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:40.831125 635 server_base.cc:1048] running on GCE node
14:10:42.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:40.829458 645 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:42.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.018702 635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:42.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.021245 635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.022584 635 hybrid_clock.cc:648] HybridClock initialized: now 1751033442022535 us; error 64 us; skew 500 ppm
14:10:42.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.023330 635 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:42.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:42 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033442, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:42.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.028869 635 init.cc:377] Logged in from keytab as kudu/127.0.133.61@KRBTEST.COM (short username kudu)
14:10:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.035054 635 webserver.cc:469] Webserver started at http://127.0.133.61:45285/ using document root <none> and password file <none>
14:10:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.036293 635 fs_manager.cc:362] Metadata directory not provided
14:10:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.036579 635 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.037104 635 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.043143 635 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/instance:
14:10:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:10:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:10:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.044291 635 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal/instance:
14:10:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:10:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:10:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.051225 635 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
14:10:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.056303 655 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.057247 635 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
14:10:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.057544 635 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:10:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.057881 635 fs_report.cc:389] FS layout report
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/data
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.109424 635 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:42.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.114684 635 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:42.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.115058 635 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:42.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.183491 635 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.61:41349
14:10:42.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.183576 706 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.61:41349 every 8 connection(s)
14:10:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.186122 635 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.186619 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 635
14:10:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.186935 532 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal/instance
14:10:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.192343 707 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:42.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.133.60@KRBTEST.COM; defaulting to no policy
14:10:42.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.133.60@KRBTEST.COM" created.
14:10:42.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.202958 707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:10:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab.
14:10:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.133.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab.
14:10:42.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:42 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033442, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.226346 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:10:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:42.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.237831 707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:42 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033442, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.61@KRBTEST.COM
14:10:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.263262 707 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.266285 657 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.275050 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.324570 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 1
14:10:42.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.327932 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.387806 516 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1a0 after lost signal to thread 391
14:10:42.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.389003 516 debug-util.cc:398] Leaking SignalData structure 0x7b08000b89e0 after lost signal to thread 519
14:10:42.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.396346 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 2
14:10:42.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.404318 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.495733 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 3
14:10:42.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.509296 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.633329 713 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.634096 713 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:42.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.635201 713 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.652699 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 4
14:10:42.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.657662 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.674412 713 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:42.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.674736 713 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.675145 713 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.675452 713 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.675725 713 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.676000 713 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.676251 713 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.712448 713 master_runner.cc:386] Master server non-default flags:
14:10:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:42.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:42.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.714052 713 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.715157 713 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.725878 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } attempt: 7
14:10:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.733497 724 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:42 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033440, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.61@KRBTEST.COM
14:10:42.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.768253 629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } has no permanent_uuid. Determining permanent_uuid...
14:10:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.788486 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.847044 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 1
14:10:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.853827 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.935323 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 2
14:10:42.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.941923 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:42.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:42.957056 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 5
14:10:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.962410 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:43.027382 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 3
14:10:43.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:43.033048 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:43.173483 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 4
14:10:43.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:43.180546 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:43.464080 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 5
14:10:43.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:43.471421 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:43.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:43.509835 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 6
14:10:43.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:43.555078 707 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:44.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.024025 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 6
14:10:44.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:44.056967 629 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.742058 723 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:42.734397 726 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:44.217931 725 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1481 milliseconds
14:10:44.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.218108 713 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.219642 713 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:44.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.222911 713 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.224365 713 hybrid_clock.cc:648] HybridClock initialized: now 1751033444224280 us; error 87 us; skew 500 ppm
14:10:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.225498 713 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.232882 713 init.cc:377] Logged in from keytab as kudu/127.0.133.60@KRBTEST.COM (short username kudu)
14:10:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.242072 713 webserver.cc:469] Webserver started at http://127.0.133.60:45467/ using document root <none> and password file <none>
14:10:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.243381 713 fs_manager.cc:362] Metadata directory not provided
14:10:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.243683 713 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:44.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.244271 713 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:44.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.251273 713 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/instance:
14:10:44.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:10:44.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:10:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.252926 713 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal/instance:
14:10:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:10:44.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:10:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.267906 713 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.006s sys 0.007s
14:10:44.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.277696 741 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.278888 713 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.004s
14:10:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.279219 713 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:10:44.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.279604 713 fs_report.cc:389] FS layout report
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/data
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:44.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:44.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:44.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.352288 713 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.357849 713 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:44.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.358357 713 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.448273 713 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.60:36057
14:10:44.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.448426 793 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.60:36057 every 8 connection(s)
14:10:44.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.450954 713 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:44.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.457190 794 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.461346 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 713
14:10:44.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.461786 532 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal/instance
14:10:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.473955 794 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
14:10:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033438, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:10:44.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.536558 794 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } has no permanent_uuid. Determining permanent_uuid...
14:10:44.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for kudu/127.0.133.61@KRBTEST.COM
14:10:44.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033438, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.133.61@KRBTEST.COM
14:10:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.568320 794 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } has no permanent_uuid. Determining permanent_uuid...
14:10:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:10:44.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033438, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.597872 794 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap starting.
14:10:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.602275 532 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
14:10:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.605379 794 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Neither blocks nor log segments found. Creating new log.
14:10:44.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.607676 794 log.cc:826] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Log is configured to *not* fsync() on all Append() calls
14:10:44.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:44.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:10:44.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:10:44.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:10:44.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.133.62"
port: 34635
}
bound_http_address {
host: "127.0.133.62"
port: 38243
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.133.61"
port: 41349
}
bound_http_address {
host: "127.0.133.61"
port: 45285
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.133.60"
port: 36057
}
bound_http_address {
host: "127.0.133.60"
port: 45467
}
}
}
14:10:44.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:10:44.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.613130 794 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: No bootstrap required, opened a new log
14:10:44.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:44.614 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:10:44.614 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:10:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.615669 707 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 7
14:10:44.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033442, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:10:44.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.638700 794 raft_consensus.cc:357] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.639685 794 raft_consensus.cc:383] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:44.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.640014 794 raft_consensus.cc:738] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f89c93328f3f430e8045205e48e3f6d5, State: Initialized, Role: FOLLOWER
14:10:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.641090 794 consensus_queue.cc:260] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.645908 807 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.646744 807 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:10:44.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.647744 794 sys_catalog.cc:564] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: configured and running, proceeding with master startup.
14:10:44.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:44.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.663439 707 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap starting.
14:10:44.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:10:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.671917 707 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Neither blocks nor log segments found. Creating new log.
14:10:44.673 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.674644 707 log.cc:826] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Log is configured to *not* fsync() on all Append() calls
14:10:44.675 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc
14:10:44.676 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:10:44.681 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
14:10:44.682 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
14:10:44.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.685542 707 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: No bootstrap required, opened a new log
14:10:44.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.702899 807 raft_consensus.cc:491] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:44.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.703444 807 raft_consensus.cc:513] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.708467 807 leader_election.cc:290] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a50e2c12f0b4410c9b930ba9d7b4a092 (127.0.133.62:34635), af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349)
14:10:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.714416 707 raft_consensus.cc:357] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.715387 707 raft_consensus.cc:383] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:44.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.716365 707 raft_consensus.cc:738] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af3deae292a8413c8c9498c2b33b9ef6, State: Initialized, Role: FOLLOWER
14:10:44.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.717532 707 consensus_queue.cc:260] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.741919 707 sys_catalog.cc:564] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: configured and running, proceeding with master startup.
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.742065 682 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f89c93328f3f430e8045205e48e3f6d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6" is_pre_election: true
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.742833 682 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f89c93328f3f430e8045205e48e3f6d5 in term 0.
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:44 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
14:10:44.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.744339 742 leader_election.cc:304] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5; no voters:
14:10:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.740782 815 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.745227 807 raft_consensus.cc:2802] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:10:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.745340 815 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:10:44.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.745559 807 raft_consensus.cc:491] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:44.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.745863 807 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 0 FOLLOWER]: Advancing to term 1
14:10:44.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.749248 822 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:44.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.752666 807 raft_consensus.cc:513] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.755798 682 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f89c93328f3f430e8045205e48e3f6d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:10:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.756367 682 raft_consensus.cc:3058] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 0 FOLLOWER]: Advancing to term 1
14:10:44.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:44.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.760854 807 leader_election.cc:290] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [CANDIDATE]: Term 1 election: Requested vote from peers a50e2c12f0b4410c9b930ba9d7b4a092 (127.0.133.62:34635), af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349)
14:10:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.766115 682 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f89c93328f3f430e8045205e48e3f6d5 in term 1.
14:10:44.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.768389 742 leader_election.cc:304] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5; no voters:
14:10:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.770364 807 raft_consensus.cc:2802] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 FOLLOWER]: Leader election won for term 1
14:10:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.786351 807 raft_consensus.cc:695] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 LEADER]: Becoming Leader. State: Replica: f89c93328f3f430e8045205e48e3f6d5, State: Running, Role: LEADER
14:10:44.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.789145 807 consensus_queue.cc:237] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:44.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.811192 827 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f89c93328f3f430e8045205e48e3f6d5. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.811899 827 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: LEADER
14:10:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.764920 713 master_runner.cc:186] Error getting master registration for 127.0.133.62:34635: OK, instance_id { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" instance_seqno: 1751033440353891 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.819473 713 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:44.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.821998 831 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:44.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.843950 831 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:44.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.860723 682 raft_consensus.cc:1273] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Refusing update from remote peer f89c93328f3f430e8045205e48e3f6d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:44.869545 747 tablet.cc:2378] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.869699 827 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:44.873669 661 tablet.cc:2378] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.876611 838 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:44.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.894726 815 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f89c93328f3f430e8045205e48e3f6d5. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.895668 815 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:10:44.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.906103 839 mvcc.cc:204] Tried to move back new op lower bound from 7172232990133903360 to 7172232989890625536. Current Snapshot: MvccSnapshot[applied={T|T < 7172232990133903360}]
14:10:44.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.915724 831 catalog_manager.cc:1349] Generated new cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:44.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.916009 831 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:44.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.916359 827 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.917200 827 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: LEADER
14:10:44.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.919202 815 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:44.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.919844 815 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:10:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.880195 635 master_runner.cc:186] Error getting master registration for 127.0.133.62:34635: OK, instance_id { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" instance_seqno: 1751033440353891 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:44.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.934659 635 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:44.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.969957 831 catalog_manager.cc:1372] Generated new certificate authority record
14:10:44.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.971822 831 catalog_manager.cc:1506] Loading token signing keys...
14:10:44.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.998517 831 catalog_manager.cc:5955] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Generated new TSK 0
14:10:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:44.999513 831 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:45.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.090354 629 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } attempt: 7
14:10:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:45 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033440, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:10:45.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.134546 629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap starting.
14:10:45.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.140048 629 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Neither blocks nor log segments found. Creating new log.
14:10:45.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.141741 629 log.cc:826] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Log is configured to *not* fsync() on all Append() calls
14:10:45.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.145726 629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: No bootstrap required, opened a new log
14:10:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.163455 629 raft_consensus.cc:357] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.164079 629 raft_consensus.cc:383] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:45.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.164350 629 raft_consensus.cc:738] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Initialized, Role: FOLLOWER
14:10:45.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.165131 629 consensus_queue.cc:260] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:45.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.167841 845 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:45.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.168740 845 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:45.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.169584 629 sys_catalog.cc:564] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: configured and running, proceeding with master startup.
14:10:45.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.170133 601 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 0 FOLLOWER]: Advancing to term 1
14:10:45.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.170815 602 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f89c93328f3f430e8045205e48e3f6d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:45.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.172537 603 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f89c93328f3f430e8045205e48e3f6d5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" is_pre_election: true
14:10:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.177688 601 raft_consensus.cc:1273] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Refusing update from remote peer f89c93328f3f430e8045205e48e3f6d5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.180418 807 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:45.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.266214 854 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.266805 854 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.271101 845 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f89c93328f3f430e8045205e48e3f6d5. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.271826 845 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:45.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.299042 861 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:45.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.299517 861 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: loading cluster ID for follower catalog manager: success
14:10:45.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.306721 861 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: acquiring CA information for follower catalog manager: success
14:10:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.312112 861 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.566390 807 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f89c93328f3f430e8045205e48e3f6d5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.567104 807 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: LEADER
14:10:45.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.885111 838 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:45.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.885396 838 catalog_manager.cc:1554] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: loading cluster ID for follower catalog manager: success
14:10:45.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.892519 838 catalog_manager.cc:1576] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: acquiring CA information for follower catalog manager: success
14:10:45.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:45.897765 838 catalog_manager.cc:1604] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:47.759 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.62:34635
14:10:47.759 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:10:47.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:47.760430 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 559
14:10:47.789 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:47.789 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.61:41349
14:10:47.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:10:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:47.790289 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 635
14:10:47.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:47.818 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.60:36057
14:10:47.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:10:47.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:47.818818 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 713
14:10:47.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:47.847 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.62:34635
14:10:47.847 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:47.848735 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:47.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.138036 872 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:48.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.138495 872 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:48.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.139003 872 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.167209 872 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.167421 872 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.167659 872 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.167837 872 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.168022 872 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.168188 872 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.168314 872 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:48.201685 872 master_runner.cc:386] Master server non-default flags:
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:48.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:48.202785 872 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:48.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:48.203723 872 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:48.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.217242 878 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:49.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.217311 879 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:49.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:48.218303 881 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:49.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:48.219364 872 server_base.cc:1048] running on GCE node
14:10:49.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.397346 872 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.399932 872 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:49.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.401271 872 hybrid_clock.cc:648] HybridClock initialized: now 1751033449401222 us; error 59 us; skew 500 ppm
14:10:49.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.402107 872 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:49 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033449, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.407804 872 init.cc:377] Logged in from keytab as kudu/127.0.133.62@KRBTEST.COM (short username kudu)
14:10:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.412899 872 webserver.cc:469] Webserver started at http://127.0.133.62:38243/ using document root <none> and password file <none>
14:10:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.413831 872 fs_manager.cc:362] Metadata directory not provided
14:10:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.414033 872 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.422047 872 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
14:10:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.426542 889 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.427652 872 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.427946 872 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.430025 872 fs_report.cc:389] FS layout report
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/data
14:10:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:49.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.482842 872 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.488379 872 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.488794 872 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.562584 872 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.62:34635
14:10:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.562675 940 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.62:34635 every 8 connection(s)
14:10:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.565193 872 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.566578 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 872
14:10:49.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:49.567 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.61:41349
14:10:49.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.569135 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:49.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.578179 941 sys_catalog.cc:263] Verifying existing consensus state
14:10:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.583384 941 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap starting.
14:10:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.619396 941 log.cc:826] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Log is configured to *not* fsync() on all Append() calls
14:10:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.633615 941 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:49.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.634560 941 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap complete.
14:10:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.655500 941 raft_consensus.cc:357] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.657651 941 raft_consensus.cc:738] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Initialized, Role: FOLLOWER
14:10:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.658341 941 consensus_queue.cc:260] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.661404 945 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.662397 945 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.663196 941 sys_catalog.cc:564] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: configured and running, proceeding with master startup.
14:10:49.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.692505 956 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.692780 956 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: loading cluster ID for follower catalog manager: success
14:10:49.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.700063 956 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: acquiring CA information for follower catalog manager: success
14:10:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.705346 956 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:49 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033449, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.740314 872 master_runner.cc:186] Error getting master registration for 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111),
14:10:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.766906 872 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:49.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.892272 943 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.893043 943 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:49.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.894034 943 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:49.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.939438 943 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:49.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.939709 943 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.940112 943 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.940376 943 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.940663 943 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:49.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.940922 943 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:49.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.941097 943 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.979358 943 master_runner.cc:386] Master server non-default flags:
14:10:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:49.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:49.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.980935 943 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:49.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.982120 943 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:49.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.998855 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.215195 972 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.215770 972 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:51.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.225083 972 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:10:51.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.231464 891 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:51.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.238423 890 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349): Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:10:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.239073 891 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:51.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.239355 891 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092; no voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5
14:10:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.240561 972 raft_consensus.cc:2747] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:10:51.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.999011 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:51.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:49.999506 943 server_base.cc:1048] running on GCE node
14:10:51.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:49.999722 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.248582 943 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.251238 943 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.252580 943 hybrid_clock.cc:648] HybridClock initialized: now 1751033451252549 us; error 40 us; skew 500 ppm
14:10:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.253347 943 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:51.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:51 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033451, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:51.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.267596 943 init.cc:377] Logged in from keytab as kudu/127.0.133.61@KRBTEST.COM (short username kudu)
14:10:51.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.279691 943 webserver.cc:469] Webserver started at http://127.0.133.61:45285/ using document root <none> and password file <none>
14:10:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.280807 943 fs_manager.cc:362] Metadata directory not provided
14:10:51.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.281196 943 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.290757 943 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
14:10:51.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.295603 981 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:51.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.296720 943 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.297004 943 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.299062 943 fs_report.cc:389] FS layout report
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/data
14:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.360052 943 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:51.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.365772 943 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:51.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.366230 943 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.457127 943 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.61:41349
14:10:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.457228 1032 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.61:41349 every 8 connection(s)
14:10:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.459879 943 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:51.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.465291 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 943
14:10:51.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:51.465 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.60:36057
14:10:51.466 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:10:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.467242 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:10:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.473162 1033 sys_catalog.cc:263] Verifying existing consensus state
14:10:51.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.480731 1033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap starting.
14:10:51.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.531086 1033 log.cc:826] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Log is configured to *not* fsync() on all Append() calls
14:10:51.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.554893 1033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:51.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.555648 1033 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap complete.
14:10:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.586263 1033 raft_consensus.cc:357] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.588464 1033 raft_consensus.cc:738] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: af3deae292a8413c8c9498c2b33b9ef6, State: Initialized, Role: FOLLOWER
14:10:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.589358 1033 consensus_queue.cc:260] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.592167 1037 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.593936 1037 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:10:51.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.594162 1033 sys_catalog.cc:564] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: configured and running, proceeding with master startup.
14:10:51.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.634634 1048 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:51.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.634902 1048 catalog_manager.cc:1554] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: loading cluster ID for follower catalog manager: success
14:10:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:51 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033451, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:51.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.648352 1048 catalog_manager.cc:1576] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: acquiring CA information for follower catalog manager: success
14:10:51.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.677676 1048 catalog_manager.cc:1604] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.722553 943 master_runner.cc:186] Error getting master registration for 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111),
14:10:51.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.749099 943 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:51.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.897080 1035 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:51.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.897979 1035 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.899067 1035 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.940409 1035 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.940712 1035 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.941110 1035 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.941378 1035 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.941663 1035 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.941965 1035 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.942174 1035 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.979514 1035 master_runner.cc:386] Master server non-default flags:
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:51.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:51.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.981280 1035 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:51.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:51.982620 1035 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:52.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:51.999485 1058 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.417572 1066 raft_consensus.cc:491] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:53.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.418154 1065 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.418541 1066 raft_consensus.cc:513] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:53.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.419116 1065 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.435942 1066 leader_election.cc:290] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers a50e2c12f0b4410c9b930ba9d7b4a092 (127.0.133.62:34635), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:10:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.475306 1065 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:10:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:53.484277 983 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:53.509424 983 leader_election.cc:336] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:53.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.510684 916 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af3deae292a8413c8c9498c2b33b9ef6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" is_pre_election: true
14:10:53.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.511409 916 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af3deae292a8413c8c9498c2b33b9ef6 in term 1.
14:10:53.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:53.513659 891 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:53.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.513533 985 leader_election.cc:304] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters: f89c93328f3f430e8045205e48e3f6d5
14:10:53.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.515089 1066 raft_consensus.cc:2802] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:10:53.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.515522 1066 raft_consensus.cc:491] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:53.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.515929 1066 raft_consensus.cc:3058] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 1 FOLLOWER]: Advancing to term 2
14:10:53.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.536132 1066 raft_consensus.cc:513] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.539628 916 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af3deae292a8413c8c9498c2b33b9ef6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.540289 916 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 1 FOLLOWER]: Advancing to term 2
14:10:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.544945 1008 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6" is_pre_election: true
14:10:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.546197 1008 raft_consensus.cc:2391] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a50e2c12f0b4410c9b930ba9d7b4a092 in current term 2: Already voted for candidate af3deae292a8413c8c9498c2b33b9ef6 in this term.
14:10:53.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.548133 890 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092; no voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5
14:10:53.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.557132 1066 leader_election.cc:290] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 election: Requested vote from peers a50e2c12f0b4410c9b930ba9d7b4a092 (127.0.133.62:34635), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:10:53.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.559960 916 raft_consensus.cc:2466] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af3deae292a8413c8c9498c2b33b9ef6 in term 2.
14:10:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.561424 1065 raft_consensus.cc:2747] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:10:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:53.565732 983 leader_election.cc:336] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:10:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.569788 985 leader_election.cc:304] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters: f89c93328f3f430e8045205e48e3f6d5
14:10:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.571219 1066 raft_consensus.cc:2802] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Leader election won for term 2
14:10:53.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.588524 1066 raft_consensus.cc:695] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 LEADER]: Becoming Leader. State: Replica: af3deae292a8413c8c9498c2b33b9ef6, State: Running, Role: LEADER
14:10:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.589906 1066 consensus_queue.cc:237] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:53.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.609812 1073 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader af3deae292a8413c8c9498c2b33b9ef6. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:53.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.611097 1073 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: LEADER
14:10:53.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.619527 1075 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:53.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.630925 1075 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.636835 1075 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:53.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.637315 1075 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:53.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.643894 1075 catalog_manager.cc:1506] Loading token signing keys...
14:10:53.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.649891 1075 catalog_manager.cc:5966] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Loaded TSK: 0
14:10:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.651552 1075 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:52.000082 1059 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.749126 1035 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:52.002822 1061 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:53.749079 1060 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1750 milliseconds
14:10:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.755646 1035 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.758750 1035 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:53.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.760253 1035 hybrid_clock.cc:648] HybridClock initialized: now 1751033453760213 us; error 59 us; skew 500 ppm
14:10:53.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.761039 1035 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:53 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033453, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.767095 1035 init.cc:377] Logged in from keytab as kudu/127.0.133.60@KRBTEST.COM (short username kudu)
14:10:53.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.772939 1035 webserver.cc:469] Webserver started at http://127.0.133.60:45467/ using document root <none> and password file <none>
14:10:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.774067 1035 fs_manager.cc:362] Metadata directory not provided
14:10:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.774354 1035 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.782883 1035 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
14:10:53.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.787758 1081 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.788867 1035 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.789242 1035 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.791029 1035 fs_report.cc:389] FS layout report
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/data
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:53.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:53.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.851104 1035 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.856873 1035 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.857318 1035 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.925379 1035 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.60:36057
14:10:53.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.925451 1132 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.60:36057 every 8 connection(s)
14:10:53.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.928174 1035 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:10:53.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.937095 1133 sys_catalog.cc:263] Verifying existing consensus state
14:10:53.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.937651 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1035
14:10:53.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:53.939 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.133.62:34635(127.0.133.62:34635)] connecting to peer
14:10:53.939 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.133.61:41349(127.0.133.61:41349)] connecting to peer
14:10:53.940 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.133.60:36057(127.0.133.60:36057)] connecting to peer
14:10:53.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.942205 1133 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap starting.
14:10:53.944 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.133.62:34635(127.0.133.62:34635)] Successfully connected to peer
14:10:53.946 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.133.60:36057(127.0.133.60:36057)] Successfully connected to peer
14:10:53.946 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.133.61:41349(127.0.133.61:41349)] Successfully connected to peer
14:10:53.950 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:10:53.950 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc
14:10:53.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:53 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:53.954 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
14:10:53.955 [ERROR - kudu-nio-1] (SecurityContext.java:246) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
14:10:53.956 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.133.61
14:10:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:53 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033444, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.133.61@KRBTEST.COM
14:10:53.984 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.133.62
14:10:53.989 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.133.60
14:10:53.992 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x9b9fb3c2, L:/127.0.0.1:51318 - R:/127.0.133.61:41349] using SASL/GSSAPI
14:10:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:53.994683 1133 log.cc:826] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Log is configured to *not* fsync() on all Append() calls
14:10:53.999 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
14:10:54.000 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=62), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.133.62:34635, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.133.61:41349, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.133.60:36057, [60ms] Sub RPC ConnectToMaster: received response from server master-127.0.133.61:41349: OK, deferred=null) after lookup
14:10:54.005 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.133.62:34635(127.0.133.62:34635)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:10:54.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.133.62:34635: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:10:54.006 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.133.62:34635(127.0.133.62:34635)] handling channelInactive
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:54.007863 1139 negotiation.cc:337] Failed RPC negotiation. Trace:
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.944278 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:54252 (local address 127.0.133.62:34635)
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.981922 (+ 37644us) server_negotiation.cc:206] Beginning negotiation
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.981959 (+ 37us) server_negotiation.cc:399] Waiting for connection header
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.982014 (+ 55us) server_negotiation.cc:407] Connection header received
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.982144 (+ 130us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.982170 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.982375 (+ 205us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.982557 (+ 182us) server_negotiation.cc:226] Negotiated authn=SASL
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.986627 (+ 4070us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.988340 (+ 1713us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.003799 (+ 15459us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.004328 (+ 529us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.006601 (+ 2273us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:54252: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:54252): Cannot send after transport endpoint shutdown (error 108)
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":36903,"thread_start_us":36981,"threads_started":1}
14:10:54.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.133.60:36057(127.0.133.60:36057)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:10:54.012 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.133.60:36057: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:10:54.013 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.133.60:36057(127.0.133.60:36057)] handling channelInactive
14:10:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.014751 1133 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:54.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.015702 1133 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap complete.
14:10:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:54.017695 1138 negotiation.cc:337] Failed RPC negotiation. Trace:
14:10:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.945309 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37300 (local address 127.0.133.60:36057)
14:10:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.974678 (+ 29369us) server_negotiation.cc:206] Beginning negotiation
14:10:54.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.974716 (+ 38us) server_negotiation.cc:399] Waiting for connection header
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.974785 (+ 69us) server_negotiation.cc:407] Connection header received
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.987580 (+ 12795us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.987654 (+ 74us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.988209 (+ 555us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.988660 (+ 451us) server_negotiation.cc:226] Negotiated authn=SASL
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.992858 (+ 4198us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:53.995105 (+ 2247us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.013593 (+ 18488us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.014537 (+ 944us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:10:54.015779 (+ 1242us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37300: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
14:10:54.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":28667,"thread_start_us":28244,"threads_started":1}
14:10:54.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.035830 1133 raft_consensus.cc:357] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:54.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.038589 1133 raft_consensus.cc:738] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: f89c93328f3f430e8045205e48e3f6d5, State: Initialized, Role: FOLLOWER
14:10:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.039428 1133 consensus_queue.cc:260] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.042064 1141 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.042830 1141 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.043622 1133 sys_catalog.cc:564] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: configured and running, proceeding with master startup.
14:10:54.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:54 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033451, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:10:54.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.080137 1152 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:54.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.080387 1152 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: loading cluster ID for follower catalog manager: success
14:10:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:54 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033453, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:54.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.082523 1108 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 1 FOLLOWER]: Advancing to term 2
14:10:54.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.085479 1152 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: acquiring CA information for follower catalog manager: success
14:10:54.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.092684 1108 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 2 FOLLOWER]: Refusing update from remote peer af3deae292a8413c8c9498c2b33b9ef6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:10:54.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.094204 1073 consensus_queue.cc:1035] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:10:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.096503 1152 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:54.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.107023 1141 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader af3deae292a8413c8c9498c2b33b9ef6. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.107689 1141 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.109565 1141 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.110162 1141 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.114576 1073 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.115253 1073 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: LEADER
14:10:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.115796 916 raft_consensus.cc:1273] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Refusing update from remote peer af3deae292a8413c8c9498c2b33b9ef6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:10:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.117201 1066 consensus_queue.cc:1035] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:10:54.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.138355 1162 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.139070 1162 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.141376 1159 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: New leader af3deae292a8413c8c9498c2b33b9ef6. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.142015 1159 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.657692 1166 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "af3deae292a8413c8c9498c2b33b9ef6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:54.658422 1166 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: LEADER
14:10:57.053 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.62:34635
14:10:57.053 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:10:57.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.054086 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 872
14:10:57.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:57.080 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.61:41349
14:10:57.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:10:57.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.080906 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 943
14:10:57.106 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.133.61:41349(127.0.133.61:41349)] handling channelInactive
14:10:57.107 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.133.61:41349(127.0.133.61:41349)] cleaning up while in state READY due to: connection closed
14:10:57.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:57.107 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.60:36057
14:10:57.107 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:10:57.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.108119 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1035
14:10:57.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:57.133 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.62:34635
14:10:57.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.134650 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:57.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:57.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:57.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.436347 1172 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:57.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.436859 1172 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:57.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.437407 1172 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.468724 1172 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.469012 1172 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.469386 1172 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:57.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.469669 1172 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.469938 1172 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.470197 1172 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:57.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.470397 1172 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.505573 1172 master_runner.cc:386] Master server non-default flags:
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:57.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:57.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.507139 1172 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.508384 1172 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:57.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.522812 1178 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.522871 1179 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:57.523105 1181 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:57.525313 1172 server_base.cc:1048] running on GCE node
14:10:58.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.673223 1172 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.676200 1172 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:58.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.677597 1172 hybrid_clock.cc:648] HybridClock initialized: now 1751033458677556 us; error 55 us; skew 500 ppm
14:10:58.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.678433 1172 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:58.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:58 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033458, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:10:58.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.687163 1172 init.cc:377] Logged in from keytab as kudu/127.0.133.62@KRBTEST.COM (short username kudu)
14:10:58.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.692397 1172 webserver.cc:469] Webserver started at http://127.0.133.62:38243/ using document root <none> and password file <none>
14:10:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.693282 1172 fs_manager.cc:362] Metadata directory not provided
14:10:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.693473 1172 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.700562 1172 fs_manager.cc:714] Time spent opening directory manager: real 0.004s user 0.006s sys 0.001s
14:10:58.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.704603 1191 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.705495 1172 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.000s sys 0.004s
14:10:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.705787 1172 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.707476 1172 fs_report.cc:389] FS layout report
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/data
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:58.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:58.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.760111 1172 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.765209 1172 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:58.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.765637 1172 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.839865 1172 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.62:34635
14:10:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.839953 1242 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.62:34635 every 8 connection(s)
14:10:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.842571 1172 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:10:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.851378 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1172
14:10:58.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:10:58.852 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.61:41349
14:10:58.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:10:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.852365 1243 sys_catalog.cc:263] Verifying existing consensus state
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.854077 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:58.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:58.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:58.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:10:58.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.858848 1243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap starting.
14:10:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.899334 1243 log.cc:826] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Log is configured to *not* fsync() on all Append() calls
14:10:58.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.914364 1243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:10:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.915091 1243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap complete.
14:10:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.936085 1243 raft_consensus.cc:357] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:58.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.938369 1243 raft_consensus.cc:738] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Initialized, Role: FOLLOWER
14:10:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.939282 1243 consensus_queue.cc:260] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:10:58.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.942139 1247 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:10:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.942960 1247 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:10:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.943820 1243 sys_catalog.cc:564] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: configured and running, proceeding with master startup.
14:10:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.972852 1258 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:10:58.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.973323 1258 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: loading cluster ID for follower catalog manager: success
14:10:58.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.981455 1258 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: acquiring CA information for follower catalog manager: success
14:10:58.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:58.987198 1258 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:10:58 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033458, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:10:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.018972 1172 master_runner.cc:186] Error getting master registration for 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111),
14:10:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.046147 1172 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.198177 1245 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.198690 1245 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.199245 1245 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.229805 1245 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:10:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230023 1245 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:59.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230276 1245 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230465 1245 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230659 1245 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230832 1245 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.230960 1245 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.273976 1245 master_runner.cc:386] Master server non-default flags:
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:10:59.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:59.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.275557 1245 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.277063 1245 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:59.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.310223 1267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:00.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.146435 1273 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.147020 1273 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.149883 1273 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:00.160619 1193 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:00.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:00.171471 1193 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:00.173395 1192 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349): Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:11:00.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.174013 1192 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092; no voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5
14:11:00.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.175022 1273 raft_consensus.cc:2747] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
14:11:00.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:10:59.311724 1268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:00.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:10:59.314095 1245 server_base.cc:1048] running on GCE node
14:11:00.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:00.561089 1270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:00.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.564088 1245 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.567122 1245 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:00.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.568539 1245 hybrid_clock.cc:648] HybridClock initialized: now 1751033460568486 us; error 49 us; skew 500 ppm
14:11:00.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.569283 1245 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:00.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:00 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033460, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:00.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.574862 1245 init.cc:377] Logged in from keytab as kudu/127.0.133.61@KRBTEST.COM (short username kudu)
14:11:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.580570 1245 webserver.cc:469] Webserver started at http://127.0.133.61:45285/ using document root <none> and password file <none>
14:11:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.581450 1245 fs_manager.cc:362] Metadata directory not provided
14:11:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.581648 1245 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.589605 1245 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
14:11:00.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.593796 1283 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:00.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.594851 1245 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.595184 1245 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.597093 1245 fs_report.cc:389] FS layout report
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/data
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:00.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:00.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:00.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:00.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.649207 1245 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.654700 1245 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.655176 1245 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.740449 1245 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.61:41349
14:11:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.740558 1334 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.61:41349 every 8 connection(s)
14:11:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.743100 1245 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:11:00.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.749053 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1245
14:11:00.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:00.749 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.60:36057
14:11:00.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.751480 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:00.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:00.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.756984 1335 sys_catalog.cc:263] Verifying existing consensus state
14:11:00.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.764520 1335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap starting.
14:11:00.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.821122 1335 log.cc:826] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Log is configured to *not* fsync() on all Append() calls
14:11:00.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.843235 1335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:00.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.844545 1335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap complete.
14:11:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.885031 1335 raft_consensus.cc:357] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.888244 1335 raft_consensus.cc:738] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: af3deae292a8413c8c9498c2b33b9ef6, State: Initialized, Role: FOLLOWER
14:11:00.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.889297 1335 consensus_queue.cc:260] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:00.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.903036 1339 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:00.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.906831 1339 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:00.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.904929 1335 sys_catalog.cc:564] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: configured and running, proceeding with master startup.
14:11:00.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.948732 1350 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:00.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.949162 1350 catalog_manager.cc:1554] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: loading cluster ID for follower catalog manager: success
14:11:00.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.961491 1350 catalog_manager.cc:1576] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: acquiring CA information for follower catalog manager: success
14:11:00.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:00.967197 1350 catalog_manager.cc:1604] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:00 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033460, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:11:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.010111 1245 master_runner.cc:186] Error getting master registration for 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111),
14:11:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.037407 1245 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:01.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.132138 1337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:01.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.132861 1337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:01.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.133576 1337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.165005 1337 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.165218 1337 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.165473 1337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.165671 1337 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.165966 1337 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.166261 1337 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.166472 1337 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.205125 1337 master_runner.cc:386] Master server non-default flags:
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.206694 1337 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:01.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.207984 1337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.224097 1361 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.708938 1366 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:01.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.709596 1366 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:01.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.718533 1366 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.754871 1193 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.761063 1310 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6" is_pre_election: true
14:11:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.761940 1310 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a50e2c12f0b4410c9b930ba9d7b4a092 in term 2.
14:11:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.763301 1192 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters: f89c93328f3f430e8045205e48e3f6d5
14:11:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.764318 1366 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Leader pre-election won for term 3
14:11:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.764650 1366 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:01.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.764947 1366 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 2 FOLLOWER]: Advancing to term 3
14:11:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.774379 1366 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.776865 1310 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:11:01.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.777374 1310 raft_consensus.cc:3058] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 2 FOLLOWER]: Advancing to term 3
14:11:01.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.798336 1310 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a50e2c12f0b4410c9b930ba9d7b4a092 in term 3.
14:11:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.804872 1193 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:01.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.802284 1366 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 election: Requested vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:01.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.806860 1192 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters: f89c93328f3f430e8045205e48e3f6d5
14:11:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.807873 1366 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Leader election won for term 3
14:11:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.821056 1366 raft_consensus.cc:695] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 LEADER]: Becoming Leader. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Running, Role: LEADER
14:11:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.822108 1366 consensus_queue.cc:237] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:01.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.845755 1371 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a50e2c12f0b4410c9b930ba9d7b4a092. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:01.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.846944 1371 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: LEADER
14:11:01.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.855444 1373 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:11:01.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.863641 1373 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:11:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.868821 1373 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:01.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.869072 1373 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:11:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.874444 1373 catalog_manager.cc:1506] Loading token signing keys...
14:11:01.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.878504 1373 catalog_manager.cc:5966] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Loaded TSK: 0
14:11:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.879662 1373 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:11:02.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:02.242287 1193 consensus_peers.cc:487] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 -> Peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Couldn't send request to peer f89c93328f3f430e8045205e48e3f6d5. Status: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:02.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.390724 1310 raft_consensus.cc:1273] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Refusing update from remote peer a50e2c12f0b4410c9b930ba9d7b4a092: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
14:11:02.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.391997 1371 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
14:11:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.407949 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a50e2c12f0b4410c9b930ba9d7b4a092. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.409216 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:02.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.422472 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.423146 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:02.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.425365 1376 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.425884 1376 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: LEADER
14:11:02.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.225154 1363 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:02.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:01.227133 1337 server_base.cc:1048] running on GCE node
14:11:02.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:01.225108 1360 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:02.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.556457 1337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.559701 1337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.561267 1337 hybrid_clock.cc:648] HybridClock initialized: now 1751033462561216 us; error 54 us; skew 500 ppm
14:11:02.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.562305 1337 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:02.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:02 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033462, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.569381 1337 init.cc:377] Logged in from keytab as kudu/127.0.133.60@KRBTEST.COM (short username kudu)
14:11:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.574935 1337 webserver.cc:469] Webserver started at http://127.0.133.60:45467/ using document root <none> and password file <none>
14:11:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.576188 1337 fs_manager.cc:362] Metadata directory not provided
14:11:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.576462 1337 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:02.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.585031 1337 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
14:11:02.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.590183 1386 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:02.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.591166 1337 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.591435 1337 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.593349 1337 fs_report.cc:389] FS layout report
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/data
14:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:02.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.641762 1337 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.647145 1337 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.647495 1337 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.737282 1337 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.60:36057
14:11:02.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.737396 1438 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.60:36057 every 8 connection(s)
14:11:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.741257 1337 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.751067 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1337
14:11:02.752 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:02.752 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.133.61:41349(127.0.133.61:41349)] connecting to peer
14:11:02.754 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.133.61:41349(127.0.133.61:41349)] Successfully connected to peer
14:11:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.757535 1439 sys_catalog.cc:263] Verifying existing consensus state
14:11:02.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:02 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033458, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:11:02.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.798628 1439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap starting.
14:11:02.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.886157 1439 log.cc:826] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Log is configured to *not* fsync() on all Append() calls
14:11:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.916641 1439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:02.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.917829 1439 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap complete.
14:11:02.919 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xf061ad86, L:/127.0.0.1:38330 - R:/127.0.133.61:41349] using TOKEN/null
14:11:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.946065 1439 raft_consensus.cc:357] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:02.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.949290 1439 raft_consensus.cc:738] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: f89c93328f3f430e8045205e48e3f6d5, State: Initialized, Role: FOLLOWER
14:11:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.950109 1439 consensus_queue.cc:260] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:02.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.953054 1445 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.953809 1445 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:11:02.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.954645 1439 sys_catalog.cc:564] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: configured and running, proceeding with master startup.
14:11:02.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.955318 1413 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 2 FOLLOWER]: Advancing to term 3
14:11:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.979944 1458 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.981184 1458 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:11:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.985376 1445 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a50e2c12f0b4410c9b930ba9d7b4a092. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:02.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:02.986124 1445 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:11:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.003708 1459 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.003947 1459 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: loading cluster ID for follower catalog manager: success
14:11:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.012039 1459 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: acquiring CA information for follower catalog manager: success
14:11:03.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.017200 1459 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:03 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033462, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:11:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.138291 1376 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:03.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:03.139024 1376 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: LEADER
14:11:05.972 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.62:34635
14:11:05.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:11:05.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:05.972704 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1172
14:11:05.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:05.998 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.61:41349
14:11:05.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:11:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:05.998937 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1245
14:11:06.023 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.133.61:41349(127.0.133.61:41349)] handling channelInactive
14:11:06.023 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:06.024 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.133.60:36057
14:11:06.024 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.133.61:41349(127.0.133.61:41349)] cleaning up while in state READY due to: connection closed
14:11:06.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:11:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:06.024400 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1337
14:11:06.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:06.051 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.62:34635
14:11:06.051 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:06.052919 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:06.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:06.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.360574 1469 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.361152 1469 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.361809 1469 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.392118 1469 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:06.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.392405 1469 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.392869 1469 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.393155 1469 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.393432 1469 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.393723 1469 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:06.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.393927 1469 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:06.430176 1469 master_runner.cc:386] Master server non-default flags:
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.62.keytab
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.62
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:06.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.62:34635
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.62
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38243
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/logs
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:06.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:06.431715 1469 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:06.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:06.432883 1469 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.447355 1477 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.447362 1476 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:06.447408 1479 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:07.667722 1478 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1217 milliseconds
14:11:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.667819 1469 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.669067 1469 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:07.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.672097 1469 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.673535 1469 hybrid_clock.cc:648] HybridClock initialized: now 1751033467673487 us; error 61 us; skew 500 ppm
14:11:07.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.674404 1469 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:07.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:07 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033467, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:07.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.687994 1469 init.cc:377] Logged in from keytab as kudu/127.0.133.62@KRBTEST.COM (short username kudu)
14:11:07.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.693110 1469 webserver.cc:469] Webserver started at http://127.0.133.62:38243/ using document root <none> and password file <none>
14:11:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.693949 1469 fs_manager.cc:362] Metadata directory not provided
14:11:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.694128 1469 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.701689 1469 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
14:11:07.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.705891 1489 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:07.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.706804 1469 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
14:11:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.707108 1469 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a50e2c12f0b4410c9b930ba9d7b4a092"
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:40 on dist-test-slave-pbh5"
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.708951 1469 fs_report.cc:389] FS layout report
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/wal
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/data
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:07.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:07.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.765844 1469 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.771595 1469 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.772001 1469 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.845876 1469 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.62:34635
14:11:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.846004 1540 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.62:34635 every 8 connection(s)
14:11:07.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.850499 1469 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-0/data/info.pb
14:11:07.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.851908 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1469
14:11:07.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:07.853 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.61:41349
14:11:07.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.855696 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:11:07.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:07.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.863651 1542 sys_catalog.cc:263] Verifying existing consensus state
14:11:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.868652 1542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap starting.
14:11:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.904305 1542 log.cc:826] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Log is configured to *not* fsync() on all Append() calls
14:11:07.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.921625 1542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:07.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.922348 1542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Bootstrap complete.
14:11:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.943137 1542 raft_consensus.cc:357] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.945353 1542 raft_consensus.cc:738] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Initialized, Role: FOLLOWER
14:11:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.946002 1542 consensus_queue.cc:260] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.948351 1546 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.949338 1546 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: FOLLOWER
14:11:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.949620 1542 sys_catalog.cc:564] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: configured and running, proceeding with master startup.
14:11:07.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.980523 1557 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.980993 1557 catalog_manager.cc:1554] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: loading cluster ID for follower catalog manager: success
14:11:07.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.988245 1557 catalog_manager.cc:1576] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: acquiring CA information for follower catalog manager: success
14:11:07.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:07 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033467, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.62@KRBTEST.COM
14:11:07.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:07.994432 1557 catalog_manager.cc:1604] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:08.005254 1469 master_runner.cc:186] Error getting master registration for 127.0.133.61:41349: Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111),
14:11:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:08.023237 1469 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:08.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.194413 1544 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.194921 1544 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.195480 1544 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:08.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.227631 1544 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.227880 1544 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.228217 1544 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.228518 1544 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:08.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.228739 1544 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.228931 1544 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:08.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.229096 1544 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:11:08.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:08.267472 1544 master_runner.cc:386] Master server non-default flags:
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.61.keytab
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.61
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.61:41349
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.61
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45285
14:11:08.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/logs
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:11:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:08.271262 1544 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:08.272761 1544 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.291378 1567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:09.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:09.591185 1572 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:09.593073 1572 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:09.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:09.626466 1491 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:09.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:09.628624 1572 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:09.640718 1491 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:09.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:09.641592 1490 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349): Network error: Client connection negotiation failed: client connection to 127.0.133.61:41349: connect: Connection refused (error 111)
14:11:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:09.642058 1490 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092; no voters: af3deae292a8413c8c9498c2b33b9ef6, f89c93328f3f430e8045205e48e3f6d5
14:11:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:09.642832 1572 raft_consensus.cc:2747] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
14:11:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:09.689426 1565 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 1544
14:11:10.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.084790 1568 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1792 milliseconds
14:11:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.085789 1544 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.794s user 0.000s sys 0.002s
14:11:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.086951 1544 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.795s user 0.000s sys 0.002s
14:11:10.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:08.292201 1566 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.088755 1544 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.089458 1570 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:10.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.091756 1544 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:10.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.101047 1544 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:10.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.102600 1544 hybrid_clock.cc:648] HybridClock initialized: now 1751033470102559 us; error 44 us; skew 500 ppm
14:11:10.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.103888 1544 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:10 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033470, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.113026 1544 init.cc:377] Logged in from keytab as kudu/127.0.133.61@KRBTEST.COM (short username kudu)
14:11:10.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.121169 1544 webserver.cc:469] Webserver started at http://127.0.133.61:45285/ using document root <none> and password file <none>
14:11:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.122063 1544 fs_manager.cc:362] Metadata directory not provided
14:11:10.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.122289 1544 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.130939 1544 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
14:11:10.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.135721 1581 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:10.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.136863 1544 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.137172 1544 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:42 on dist-test-slave-pbh5"
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.139110 1544 fs_report.cc:389] FS layout report
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/wal
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/data
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:10.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:10.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.199555 1544 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.206231 1544 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.206745 1544 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.307654 1544 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.61:41349
14:11:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.307814 1632 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.61:41349 every 8 connection(s)
14:11:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.310385 1544 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-1/data/info.pb
14:11:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.310860 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1544
14:11:10.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:10.311 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.133.60:36057
14:11:10.311 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.313457 532 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:10.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.323916 1633 sys_catalog.cc:263] Verifying existing consensus state
14:11:10.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.331548 1633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap starting.
14:11:10.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.384781 1633 log.cc:826] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Log is configured to *not* fsync() on all Append() calls
14:11:10.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.406611 1633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:10.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.407565 1633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: Bootstrap complete.
14:11:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.443581 1633 raft_consensus.cc:357] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.447220 1633 raft_consensus.cc:738] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: af3deae292a8413c8c9498c2b33b9ef6, State: Initialized, Role: FOLLOWER
14:11:10.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.448159 1633 consensus_queue.cc:260] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:10.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.451103 1638 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:10.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.452498 1638 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.453194 1633 sys_catalog.cc:564] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: configured and running, proceeding with master startup.
14:11:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.507696 1649 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.507959 1649 catalog_manager.cc:1554] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: loading cluster ID for follower catalog manager: success
14:11:10.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.514911 1649 catalog_manager.cc:1576] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: acquiring CA information for follower catalog manager: success
14:11:10.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.536852 1649 catalog_manager.cc:1604] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.579844 1544 master_runner.cc:186] Error getting master registration for 127.0.133.60:36057: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111),
14:11:10.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.609982 1544 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.691176 1635 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.691860 1635 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.692539 1635 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.723857 1635 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.724089 1635 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.724511 1635 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.724790 1635 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.725044 1635 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.725287 1635 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.725474 1635 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34571/jwks.json
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.760766 1635 master_runner.cc:386] Master server non-default flags:
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.133.20:40081
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/kudu_127.0.133.60.keytab
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.133.60
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.133.60:36057
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34571/jwks.json
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.133.60
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45467
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:10.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/logs
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f813aeead00d01482b21a4c28bdb5620b28c8aaf
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 27 Jun 2025 13:43:23 UTC on 5fd53c4cbb9d
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6797
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:10.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.762338 1635 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:10.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:10.763365 1635 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:10.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.781876 1659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:11.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.250820 1663 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:11.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.251410 1663 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:11.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.297457 1663 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:11.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.321087 1608 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6" is_pre_election: true
14:11:11.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.322114 1608 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a50e2c12f0b4410c9b930ba9d7b4a092 in term 3.
14:11:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.324105 1490 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters:
14:11:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.325145 1663 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Leader pre-election won for term 4
14:11:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.325459 1663 raft_consensus.cc:491] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:11.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.325742 1663 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 3 FOLLOWER]: Advancing to term 4
14:11:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.336112 1663 raft_consensus.cc:513] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.340206 1608 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "af3deae292a8413c8c9498c2b33b9ef6"
14:11:11.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.340924 1608 raft_consensus.cc:3058] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 3 FOLLOWER]: Advancing to term 4
14:11:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.356202 1608 raft_consensus.cc:2466] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a50e2c12f0b4410c9b930ba9d7b4a092 in term 4.
14:11:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.357708 1490 leader_election.cc:304] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a50e2c12f0b4410c9b930ba9d7b4a092, af3deae292a8413c8c9498c2b33b9ef6; no voters:
14:11:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.369246 1668 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 4 FOLLOWER]: Leader election won for term 4
14:11:11.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.372826 1663 leader_election.cc:290] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 election: Requested vote from peers af3deae292a8413c8c9498c2b33b9ef6 (127.0.133.61:41349), f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057)
14:11:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.376464 1668 raft_consensus.cc:695] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [term 4 LEADER]: Becoming Leader. State: Replica: a50e2c12f0b4410c9b930ba9d7b4a092, State: Running, Role: LEADER
14:11:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.377384 1668 consensus_queue.cc:237] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:11.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:11.381989 1491 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.0.133.60:36057, user_credentials={real_user=slave}} blocked reactor thread for 67620.8us
14:11:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:11.388131 1491 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:11.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:11.391921 1491 leader_election.cc:336] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111)
14:11:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.399706 1663 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a50e2c12f0b4410c9b930ba9d7b4a092. Latest consensus state: current_term: 4 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:11.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.404161 1663 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: LEADER
14:11:11.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.413691 1671 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:11:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.434278 1671 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:11:11.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.438532 1671 catalog_manager.cc:1261] Loaded cluster ID: c11f3410576c43b5b9430e08dc139e62
14:11:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.438809 1671 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:11:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.443156 1671 catalog_manager.cc:1506] Loading token signing keys...
14:11:11.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.447047 1671 catalog_manager.cc:5966] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092: Loaded TSK: 0
14:11:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.448202 1671 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:11:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.784879 1608 raft_consensus.cc:1273] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [term 4 FOLLOWER]: Refusing update from remote peer a50e2c12f0b4410c9b930ba9d7b4a092: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
14:11:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.787281 1663 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
14:11:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:11.835088 1491 consensus_peers.cc:487] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 -> Peer f89c93328f3f430e8045205e48e3f6d5 (127.0.133.60:36057): Couldn't send request to peer f89c93328f3f430e8045205e48e3f6d5. Status: Network error: Client connection negotiation failed: client connection to 127.0.133.60:36057: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:11.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.837734 1672 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a50e2c12f0b4410c9b930ba9d7b4a092. Latest consensus state: current_term: 4 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:11.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.838301 1663 sys_catalog.cc:455] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:11.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.839093 1663 sys_catalog.cc:458] T 00000000000000000000000000000000 P a50e2c12f0b4410c9b930ba9d7b4a092 [sys.catalog]: This master's current role is: LEADER
14:11:11.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.839951 1672 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.846082 1672 sys_catalog.cc:455] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:11.847083 1672 sys_catalog.cc:458] T 00000000000000000000000000000000 P af3deae292a8413c8c9498c2b33b9ef6 [sys.catalog]: This master's current role is: FOLLOWER
14:11:12.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:10.784065 1660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:12.177713 1658 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 1635
14:11:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:12.744800 1635 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 1.963s user 0.587s sys 1.230s
14:11:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:12.746322 1635 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 1.965s user 0.587s sys 1.230s
14:11:12.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.756327 1635 server_base.cc:1048] running on GCE node
14:11:12.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:12.757979 1680 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.759865 1635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:12.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.764081 1635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:12.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.765627 1635 hybrid_clock.cc:648] HybridClock initialized: now 1751033472765582 us; error 35 us; skew 500 ppm
14:11:12.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.766847 1635 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:12.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:12 dist-test-slave-pbh5 krb5kdc[543](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033472, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:12.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.777604 1635 init.cc:377] Logged in from keytab as kudu/127.0.133.60@KRBTEST.COM (short username kudu)
14:11:12.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.783576 1635 webserver.cc:469] Webserver started at http://127.0.133.60:45467/ using document root <none> and password file <none>
14:11:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.784435 1635 fs_manager.cc:362] Metadata directory not provided
14:11:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.784698 1635 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.792905 1635 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
14:11:12.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.797623 1686 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.798710 1635 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.799033 1635 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data,/tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f89c93328f3f430e8045205e48e3f6d5"
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-27 14:10:44 on dist-test-slave-pbh5"
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.801273 1635 fs_report.cc:389] FS layout report
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/wal
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/data
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:12.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:12.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:12.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.856144 1635 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.862143 1635 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:12.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.862512 1635 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.949576 1635 rpc_server.cc:307] RPC server started. Bound to: 127.0.133.60:36057
14:11:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.949657 1737 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.0.133.60:36057 every 8 connection(s)
14:11:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.952445 1635 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/master-2/data/info.pb
14:11:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.960546 532 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu as pid 1635
14:11:12.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:12.961 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.133.61:41349(127.0.133.61:41349)] connecting to peer
14:11:12.963 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.133.61:41349(127.0.133.61:41349)] Successfully connected to peer
14:11:12.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.966346 1738 sys_catalog.cc:263] Verifying existing consensus state
14:11:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:12.974151 1738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap starting.
14:11:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:12.993496 1740 negotiation.cc:344] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:52378: token expired
14:11:12.994 [DEBUG - kudu-nio-0] (Negotiator.java:352) peer /127.0.133.61:41349 sent connection negotiation error: Not authorized: token expired
14:11:12.995 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.133.61:41349(127.0.133.61:41349)] handling channelInactive
14:11:12.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.133.61:41349(127.0.133.61:41349)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
14:11:12.997 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.0.133.62:34635(127.0.133.62:34635)] connecting to peer
14:11:13.001 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.133.62:34635(127.0.133.62:34635)] Successfully connected to peer
14:11:13.002 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.0.133.61:41349(127.0.133.61:41349)] connecting to peer
14:11:13.003 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.0.133.60:36057(127.0.133.60:36057)] connecting to peer
14:11:13.005 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.0.133.60:36057(127.0.133.60:36057)] Successfully connected to peer
14:11:13.007 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.133.61:41349(127.0.133.61:41349)] Successfully connected to peer
14:11:13.009 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:11:13.010 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLIfY9B/test-tmp/mini-kudu-cluster7796417471774054460/krb5kdc/krb5cc
14:11:13.011 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:11:13.011 [WARN - kudu-nio-0] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
14:11:13.013 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.133.61:41349(127.0.133.61:41349)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.133.62:34635(127.0.133.62:34635)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.014 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.133.61:41349: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.014 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.133.62:34635: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.014 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.133.62:34635(127.0.133.62:34635)] handling channelInactive
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:13.016120 1740 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.006710 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:52386 (local address 127.0.133.61:41349)
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.007989 (+ 1279us) server_negotiation.cc:206] Beginning negotiation
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008013 (+ 24us) server_negotiation.cc:399] Waiting for connection header
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008056 (+ 43us) server_negotiation.cc:407] Connection header received
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008180 (+ 124us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008209 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008620 (+ 411us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.009010 (+ 390us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.014893 (+ 5883us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:52386: BlockingRecv error: recv got EOF from 127.0.0.1:52386 (error 108)
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":713}
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:13.016180 1741 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.001334 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43116 (local address 127.0.133.62:34635)
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.012975 (+ 11641us) server_negotiation.cc:206] Beginning negotiation
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.012992 (+ 17us) server_negotiation.cc:399] Waiting for connection header
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.013024 (+ 32us) server_negotiation.cc:407] Connection header received
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.013120 (+ 96us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.013136 (+ 16us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.013278 (+ 142us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.013448 (+ 170us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.014981 (+ 1533us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43116: BlockingRecv error: recv got EOF from 127.0.0.1:43116 (error 108)
14:11:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11264,"thread_start_us":11010,"threads_started":1}
14:11:13.018 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.133.61:41349(127.0.133.61:41349)] handling channelInactive
14:11:13.036 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.133.60:36057(127.0.133.60:36057)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.036 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.133.60:36057: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
14:11:13.038 [DEBUG - main] (Connection.java:575) [peer master-127.0.133.61:41349(127.0.133.61:41349)] disconnecting while in state TERMINATED
14:11:13.038 [DEBUG - main] (Connection.java:575) [peer master-127.0.133.60:36057(127.0.133.60:36057)] disconnecting while in state TERMINATED
14:11:13.038 [DEBUG - main] (Connection.java:575) [peer master-127.0.133.62:34635(127.0.133.62:34635)] disconnecting while in state TERMINATED
14:11:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:13.039319 1744 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.005600 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42184 (local address 127.0.133.60:36057)
14:11:13.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.023217 (+ 17617us) server_negotiation.cc:206] Beginning negotiation
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.023266 (+ 49us) server_negotiation.cc:399] Waiting for connection header
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.023365 (+ 99us) server_negotiation.cc:407] Connection header received
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.034956 (+ 11591us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.034990 (+ 34us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.035254 (+ 264us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.035431 (+ 177us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.037588 (+ 2157us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42184: BlockingRecv error: recv got EOF from 127.0.0.1:42184 (error 108)
14:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16915,"thread_start_us":24609,"threads_started":1}
14:11:13.042 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:13.042 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.0.133.60:36057(127.0.133.60:36057)] handling channelInactive
14:11:13.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.044466 1738 log.cc:826] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Log is configured to *not* fsync() on all Append() calls
14:11:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 27 14:11:13 dist-test-slave-pbh5 krb5kdc[543](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751033467, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.133.62@KRBTEST.COM for kudu/127.0.133.60@KRBTEST.COM
14:11:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.053308 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1469
14:11:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.075824 1738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:13.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.076836 1738 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5: Bootstrap complete.
14:11:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.084631 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1544
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250627 14:11:13.088918 1742 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.008792 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42188 (local address 127.0.133.60:36057)
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.031218 (+ 22426us) server_negotiation.cc:206] Beginning negotiation
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.031239 (+ 21us) server_negotiation.cc:399] Waiting for connection header
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.031287 (+ 48us) server_negotiation.cc:407] Connection header received
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.032147 (+ 860us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.032205 (+ 58us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.032748 (+ 543us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.033166 (+ 418us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.035802 (+ 2636us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.042747 (+ 6945us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.048429 (+ 5682us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.049205 (+ 776us) server_negotiation.cc:657] Negotiated auth-only TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.052301 (+ 3096us) server_negotiation.cc:365] Received SASL_INITIATE NegotiatePB request
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.052458 (+ 157us) server_negotiation.cc:881] Received SASL_INITIATE request from client
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.052543 (+ 85us) server_negotiation.cc:892] Client requested to use mechanism: GSSAPI
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.052734 (+ 191us) server_negotiation.cc:922] calling sasl_server_start()
14:11:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.062438 (+ 9704us) server_negotiation.cc:377] Sending SASL_CHALLENGE NegotiatePB response
14:11:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0627 14:11:13.087100 (+ 24662us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42188: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:11:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21842,"thread_start_us":13424,"threads_started":1}
14:11:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.107568 1738 raft_consensus.cc:357] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.111145 1738 raft_consensus.cc:738] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: f89c93328f3f430e8045205e48e3f6d5, State: Initialized, Role: FOLLOWER
14:11:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.112140 1738 consensus_queue.cc:260] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } }
14:11:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.114873 1747 sys_catalog.cc:455] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a50e2c12f0b4410c9b930ba9d7b4a092" member_type: VOTER last_known_addr { host: "127.0.133.62" port: 34635 } } peers { permanent_uuid: "af3deae292a8413c8c9498c2b33b9ef6" member_type: VOTER last_known_addr { host: "127.0.133.61" port: 41349 } } peers { permanent_uuid: "f89c93328f3f430e8045205e48e3f6d5" member_type: VOTER last_known_addr { host: "127.0.133.60" port: 36057 } } }
14:11:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.115470 1747 sys_catalog.cc:458] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: This master's current role is: FOLLOWER
14:11:13.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.117090 1738 sys_catalog.cc:564] T 00000000000000000000000000000000 P f89c93328f3f430e8045205e48e3f6d5 [sys.catalog]: configured and running, proceeding with master startup.
14:11:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:13.120455 532 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 1635
14:11:14.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:11:14Z chronyd exiting
14:11:15.297 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.298236 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 32556
14:11:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.320968 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 32711
14:11:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.344784 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 386
14:11:15.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.368816 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 32287
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.395996 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 32357
14:11:15.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250627 14:11:15.422432 32275 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskLIfY9B/build/tsan/bin/kudu with pid 32432
14:11:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-27T14:11:16Z chronyd exiting
14:11:17.546 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:11:18.001 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: giving up after 1 attempts
14:11:18.001 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.0.133.62:34635,127.0.133.61:41349,127.0.133.60:36057). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more