14:10:48.502 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:10:48.502 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGY_x37/test-tmp
14:10:48.503 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:10:48.503 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGY_x37/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:10:48.504 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:10:48.822 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:10:48.822 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:10:48.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:48.825184 17778 mini_oidc.cc:150] Starting JWKS server
14:10:48.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:48.828799 17778 webserver.cc:469] Webserver started at http://127.0.0.1:37129/ using document root <none> and password file <none>
14:10:48.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:48.829311 17778 mini_oidc.cc:186] Starting OIDC Discovery server
14:10:48.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:48.831115 17778 webserver.cc:469] Webserver started at http://127.0.0.1:37737/ using document root <none> and password file <none>
14:10:48.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:10:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:10:48.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:10:48Z Disabled control of system clock
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:48.897135 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/logs
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data/info.pb
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:48.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.190:38563
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.190
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:48.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json with env {}
14:10:49.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.192446 17790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:49.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.193004 17790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:49.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.193606 17790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.226409 17790 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:49.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.226691 17790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.226889 17790 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.227115 17790 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.227303 17790 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:49.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.227447 17790 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37129/jwks.json
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:49.263269 17790 master_runner.cc:386] Master server non-default flags:
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.190:38563
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data/info.pb
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.190
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/logs
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:49.264569 17790 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:49.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:49.266202 17790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:49.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.281497 17799 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.281862 17797 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:50.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:49.281852 17796 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:50.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:49.282730 17790 server_base.cc:1048] running on GCE node
14:10:50.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.470857 17790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:50.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.473479 17790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:50.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.474843 17790 hybrid_clock.cc:648] HybridClock initialized: now 1751119850474796 us; error 56 us; skew 500 ppm
14:10:50.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.475620 17790 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:50.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.482318 17790 webserver.cc:469] Webserver started at http://127.17.92.190:36373/ using document root <none> and password file <none>
14:10:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.483217 17790 fs_manager.cc:362] Metadata directory not provided
14:10:50.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.483426 17790 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:50.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.483955 17790 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.488404 17790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data/instance:
14:10:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3946c743e77f4002817a63300f2adffa"
14:10:50.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:50 on dist-test-slave-n2gf"
14:10:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.489538 17790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal/instance:
14:10:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3946c743e77f4002817a63300f2adffa"
14:10:50.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:50 on dist-test-slave-n2gf"
14:10:50.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.496690 17790 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:10:50.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.502565 17806 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:50.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.503543 17790 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.503923 17790 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3946c743e77f4002817a63300f2adffa"
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:50 on dist-test-slave-n2gf"
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.504277 17790 fs_report.cc:389] FS layout report
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal
14:10:50.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data/data
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:50.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:50.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.560112 17790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:50.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.561642 17790 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:50.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.562153 17790 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.634307 17790 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.190:38563
14:10:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.634389 17857 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.190:38563 every 8 connection(s)
14:10:50.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.637012 17790 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/data/info.pb
14:10:50.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.642264 17858 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:50.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.644217 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 17790
14:10:50.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.644599 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-0/wal/instance
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.647979 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/logs
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data/info.pb
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:50.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.189:33237
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.189
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:50.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json with env {}
14:10:50.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.653891 17858 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } has no permanent_uuid. Determining permanent_uuid...
14:10:50.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.680040 17858 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } has no permanent_uuid. Determining permanent_uuid...
14:10:50.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.682734 17808 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.686964 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.736552 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 1
14:10:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.740496 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.809007 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 2
14:10:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.812896 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:50.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:50.904387 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 3
14:10:50.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.908242 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.953208 17860 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.953809 17860 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.954301 17860 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.985839 17860 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.986153 17860 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.986356 17860 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.986548 17860 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.986733 17860 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:50.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:50.986877 17860 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37129/jwks.json
14:10:51.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.022540 17860 master_runner.cc:386] Master server non-default flags:
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.189:33237
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data/info.pb
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.189
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/logs
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:51.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:10:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:10:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:51.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.023850 17860 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:51.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.025460 17860 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.041817 17869 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:51.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.051736 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 4
14:10:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.055522 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.354995 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 5
14:10:51.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.360554 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.907996 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 6
14:10:51.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.915117 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.189:33237: Network error: Client connection negotiation failed: client connection to 127.17.92.189:33237: connect: Connection refused (error 111)
14:10:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.041903 17870 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:51.042316 17872 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:52.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:51.043450 17860 server_base.cc:1048] running on GCE node
14:10:52.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.232621 17860 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:52.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.235248 17860 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:52.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.236574 17860 hybrid_clock.cc:648] HybridClock initialized: now 1751119852236552 us; error 46 us; skew 500 ppm
14:10:52.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.237324 17860 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:52.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.243681 17860 webserver.cc:469] Webserver started at http://127.17.92.189:41525/ using document root <none> and password file <none>
14:10:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.244624 17860 fs_manager.cc:362] Metadata directory not provided
14:10:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.244807 17860 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.245199 17860 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:52.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.249558 17860 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data/instance:
14:10:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e14717bfe4c4caea989247601d1934c"
14:10:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:52 on dist-test-slave-n2gf"
14:10:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.250615 17860 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal/instance:
14:10:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e14717bfe4c4caea989247601d1934c"
14:10:52.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:52 on dist-test-slave-n2gf"
14:10:52.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.257704 17860 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
14:10:52.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.263206 17881 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:52.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.264298 17860 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.264698 17860 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e14717bfe4c4caea989247601d1934c"
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:52 on dist-test-slave-n2gf"
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.265002 17860 fs_report.cc:389] FS layout report
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data/data
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:52.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:52.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:52.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.314931 17860 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.316510 17860 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.316953 17860 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:52.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.388648 17860 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.189:33237
14:10:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.388736 17932 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.189:33237 every 8 connection(s)
14:10:52.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.391335 17860 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/data/info.pb
14:10:52.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.396395 17933 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.397668 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 17860
14:10:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.398130 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-1/wal/instance
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.401165 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/logs
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data/info.pb
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.188:40655
14:10:52.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.188
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json with env {}
14:10:52.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.410897 17933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } has no permanent_uuid. Determining permanent_uuid...
14:10:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.435135 17933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } has no permanent_uuid. Determining permanent_uuid...
14:10:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.449878 17933 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } has no permanent_uuid. Determining permanent_uuid...
14:10:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.452001 17884 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.456207 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.505756 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 1
14:10:52.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.509929 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.578426 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 2
14:10:52.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.582320 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.673722 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 3
14:10:52.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.677987 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.712253 17935 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:52.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.712790 17935 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.713318 17935 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:52.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.744701 17935 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:10:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.744984 17935 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.745190 17935 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:10:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.745383 17935 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:10:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.745564 17935 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:10:52.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.745711 17935 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37129/jwks.json
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.781070 17935 master_runner.cc:386] Master server non-default flags:
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.188:40655
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37129/jwks.json
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data/info.pb
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.188
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/logs
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.782594 17935 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:52.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.784287 17935 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:52.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.798437 17946 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:52.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.821462 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 4
14:10:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.825775 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:52.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.975756 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } attempt: 7
14:10:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.987439 17858 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } has no permanent_uuid. Determining permanent_uuid...
14:10:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.993754 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.052122 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 1
14:10:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.056551 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.125375 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 5
14:10:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.129709 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.137900 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 2
14:10:53.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.142180 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.227551 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 3
14:10:53.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.233527 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.373936 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 4
14:10:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.379143 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.662503 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 5
14:10:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.667747 17858 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.677337 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 6
14:10:53.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:53.684460 17933 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.17.92.188:40655: Network error: Client connection negotiation failed: client connection to 127.17.92.188:40655: connect: Connection refused (error 111)
14:10:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.799933 17945 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:52.800544 17935 server_base.cc:1048] running on GCE node
14:10:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:52.800590 17948 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.995503 17935 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:53.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.998179 17935 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:53.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:53.999524 17935 hybrid_clock.cc:648] HybridClock initialized: now 1751119853999500 us; error 44 us; skew 500 ppm
14:10:54.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.000358 17935 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.006569 17935 webserver.cc:469] Webserver started at http://127.17.92.188:45725/ using document root <none> and password file <none>
14:10:54.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.007460 17935 fs_manager.cc:362] Metadata directory not provided
14:10:54.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.007648 17935 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:54.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.008127 17935 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.012672 17935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data/instance:
14:10:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77715668a3554c90a7b7658838713c0e"
14:10:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:54 on dist-test-slave-n2gf"
14:10:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.013860 17935 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal/instance:
14:10:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77715668a3554c90a7b7658838713c0e"
14:10:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:54 on dist-test-slave-n2gf"
14:10:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.021181 17935 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
14:10:54.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.027037 17960 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.028198 17935 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.028494 17935 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal
14:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77715668a3554c90a7b7658838713c0e"
14:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:54 on dist-test-slave-n2gf"
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.028803 17935 fs_report.cc:389] FS layout report
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data/data
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.087081 17935 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:54.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.088680 17935 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.089118 17935 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:54.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.159284 17935 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.188:40655
14:10:54.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.159364 18011 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.188:40655 every 8 connection(s)
14:10:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.161978 17935 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/data/info.pb
14:10:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.167048 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 17935
14:10:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.167433 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/master-2/wal/instance
14:10:54.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.168120 18012 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:10:54.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.186518 18012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } has no permanent_uuid. Determining permanent_uuid...
14:10:54.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.212946 18012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } has no permanent_uuid. Determining permanent_uuid...
14:10:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.220140 17858 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 6
14:10:54.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.222031 18012 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } has no permanent_uuid. Determining permanent_uuid...
14:10:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.241842 17858 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: Bootstrap starting.
14:10:54.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.243014 18012 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: Bootstrap starting.
14:10:54.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.247884 17858 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: Neither blocks nor log segments found. Creating new log.
14:10:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.249620 17858 log.cc:826] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: Log is configured to *not* fsync() on all Append() calls
14:10:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.249677 18012 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: Neither blocks nor log segments found. Creating new log.
14:10:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.252137 18012 log.cc:826] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: Log is configured to *not* fsync() on all Append() calls
14:10:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.255112 17858 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: No bootstrap required, opened a new log
14:10:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.257266 18012 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: No bootstrap required, opened a new log
14:10:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.274803 17858 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.276003 17858 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.276314 17858 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3946c743e77f4002817a63300f2adffa, State: Initialized, Role: FOLLOWER
14:10:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.276943 17858 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.277036 18012 raft_consensus.cc:357] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.277735 18012 raft_consensus.cc:383] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:54.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.277976 18012 raft_consensus.cc:738] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 77715668a3554c90a7b7658838713c0e, State: Initialized, Role: FOLLOWER
14:10:54.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.278761 18012 consensus_queue.cc:260] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.279874 18024 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.280700 18024 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.281664 17858 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: configured and running, proceeding with master startup.
14:10:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.281968 18025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.282799 18025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.283720 18012 sys_catalog.cc:564] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: configured and running, proceeding with master startup.
14:10:54.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.321797 17812 tablet.cc:2378] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:54.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.322516 18047 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:54.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.322827 18047 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.331146 18044 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:10:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.331732 18044 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:10:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:54.344084 17966 tablet.cc:2378] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:10:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.326623 17935 master_runner.cc:186] Error getting master registration for 127.17.92.189:33237: OK, instance_id { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" instance_seqno: 1751119852369241 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.364290 17935 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.317162 17790 master_runner.cc:186] Error getting master registration for 127.17.92.189:33237: OK, instance_id { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" instance_seqno: 1751119852369241 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:10:54.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.367632 17790 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:10:54.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.429477 18024 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:10:54.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.430006 18024 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.432001 18024 leader_election.cc:290] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2e14717bfe4c4caea989247601d1934c (127.17.92.189:33237), 77715668a3554c90a7b7658838713c0e (127.17.92.188:40655)
14:10:54.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.433794 17987 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3946c743e77f4002817a63300f2adffa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "77715668a3554c90a7b7658838713c0e" is_pre_election: true
14:10:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.434715 17987 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3946c743e77f4002817a63300f2adffa in term 0.
14:10:54.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.435999 17809 leader_election.cc:304] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3946c743e77f4002817a63300f2adffa, 77715668a3554c90a7b7658838713c0e; no voters:
14:10:54.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.436687 18024 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Leader pre-election won for term 1
14:10:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.436954 18024 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:10:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.437225 18024 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 0 FOLLOWER]: Advancing to term 1
14:10:54.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.443384 18024 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.446681 18024 leader_election.cc:290] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [CANDIDATE]: Term 1 election: Requested vote from peers 2e14717bfe4c4caea989247601d1934c (127.17.92.189:33237), 77715668a3554c90a7b7658838713c0e (127.17.92.188:40655)
14:10:54.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.448599 17987 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3946c743e77f4002817a63300f2adffa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "77715668a3554c90a7b7658838713c0e"
14:10:54.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.449108 17987 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 0 FOLLOWER]: Advancing to term 1
14:10:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.456982 17987 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3946c743e77f4002817a63300f2adffa in term 1.
14:10:54.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.464740 17809 leader_election.cc:304] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3946c743e77f4002817a63300f2adffa, 77715668a3554c90a7b7658838713c0e; no voters:
14:10:54.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.465708 18024 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 1 FOLLOWER]: Leader election won for term 1
14:10:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.473920 18024 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [term 1 LEADER]: Becoming Leader. State: Replica: 3946c743e77f4002817a63300f2adffa, State: Running, Role: LEADER
14:10:54.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.474896 18024 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.487586 18050 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3946c743e77f4002817a63300f2adffa. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.488410 18050 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: This master's current role is: LEADER
14:10:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.500185 18052 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:10:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.512485 18052 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:10:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.530139 17987 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [term 1 FOLLOWER]: Refusing update from remote peer 3946c743e77f4002817a63300f2adffa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:54.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.531934 18050 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [LEADER]: Connected to new peer: Peer: permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:54.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.562537 18025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3946c743e77f4002817a63300f2adffa. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.569795 18053 mvcc.cc:204] Tried to move back new op lower bound from 7172586924139110400 to 7172586923938705408. Current Snapshot: MvccSnapshot[applied={T|T < 7172586924139110400}]
14:10:54.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.570736 18025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.583145 18054 mvcc.cc:204] Tried to move back new op lower bound from 7172586924139110400 to 7172586923938705408. Current Snapshot: MvccSnapshot[applied={T|T < 7172586924139110400}]
14:10:54.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.597915 18024 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.598819 18024 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: This master's current role is: LEADER
14:10:54.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.607753 18025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.608525 18025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e [sys.catalog]: This master's current role is: FOLLOWER
14:10:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.612967 18052 catalog_manager.cc:1349] Generated new cluster ID: acd062ebff4a4db9803830157dc9e3a8
14:10:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.613278 18052 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:10:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.666204 18052 catalog_manager.cc:1372] Generated new certificate authority record
14:10:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.677381 18052 catalog_manager.cc:1506] Loading token signing keys...
14:10:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.712574 18052 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa: Generated new TSK 0
14:10:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.714637 18052 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:10:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.745100 17933 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } attempt: 7
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.755474 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/logs
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data/info.pb
14:10:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.129:0
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.129
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.129
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.799544 17933 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: Bootstrap starting.
14:10:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.810537 17933 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: Neither blocks nor log segments found. Creating new log.
14:10:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.813334 17933 log.cc:826] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: Log is configured to *not* fsync() on all Append() calls
14:10:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.834877 17933 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: No bootstrap required, opened a new log
14:10:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.901379 17933 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.903892 17933 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:10:54.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.906183 17933 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2e14717bfe4c4caea989247601d1934c, State: Initialized, Role: FOLLOWER
14:10:54.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.908746 17933 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } }
14:10:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.935070 17907 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3946c743e77f4002817a63300f2adffa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2e14717bfe4c4caea989247601d1934c"
14:10:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.936472 17907 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 0 FOLLOWER]: Advancing to term 1
14:10:54.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.935639 17908 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3946c743e77f4002817a63300f2adffa" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2e14717bfe4c4caea989247601d1934c" is_pre_election: true
14:10:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.948961 17907 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3946c743e77f4002817a63300f2adffa in term 1.
14:10:54.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.951166 17906 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [term 1 FOLLOWER]: Refusing update from remote peer 3946c743e77f4002817a63300f2adffa: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:10:54.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.931980 18063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:54.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.958479 18050 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [LEADER]: Connected to new peer: Peer: permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:10:54.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.950022 17933 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: configured and running, proceeding with master startup.
14:10:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:54.958806 18063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: This master's current role is: FOLLOWER
14:10:55.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.013365 18066 mvcc.cc:204] Tried to move back new op lower bound from 7172586924139110400 to 7172586923938705408. Current Snapshot: MvccSnapshot[applied={T|T < 7172586924139110400}]
14:10:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.041877 18063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3946c743e77f4002817a63300f2adffa. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.042646 18063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: This master's current role is: FOLLOWER
14:10:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.043228 18064 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.043987 18064 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c [sys.catalog]: This master's current role is: FOLLOWER
14:10:55.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.069072 18080 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:10:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.277298 18061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.277841 18061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:55.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.278373 18061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:55.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.312675 18061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.313637 18061 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.92.129
14:10:55.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.329536 18047 catalog_manager.cc:1261] Loaded cluster ID: acd062ebff4a4db9803830157dc9e3a8
14:10:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.330030 18047 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: loading cluster ID for follower catalog manager: success
14:10:55.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.338521 18047 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: acquiring CA information for follower catalog manager: success
14:10:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.344560 18047 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 77715668a3554c90a7b7658838713c0e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.359239 18061 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:55.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.129:0
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data/info.pb
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.129
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.129
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/logs
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.361024 18061 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:55.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.362742 18061 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:55.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.387398 18086 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.399825 18088 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3946c743e77f4002817a63300f2adffa" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3946c743e77f4002817a63300f2adffa" member_type: VOTER last_known_addr { host: "127.17.92.190" port: 38563 } } peers { permanent_uuid: "2e14717bfe4c4caea989247601d1934c" member_type: VOTER last_known_addr { host: "127.17.92.189" port: 33237 } } peers { permanent_uuid: "77715668a3554c90a7b7658838713c0e" member_type: VOTER last_known_addr { host: "127.17.92.188" port: 40655 } } }
14:10:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:55.403873 18088 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3946c743e77f4002817a63300f2adffa [sys.catalog]: This master's current role is: LEADER
14:10:56.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:56.103507 18080 catalog_manager.cc:1261] Loaded cluster ID: acd062ebff4a4db9803830157dc9e3a8
14:10:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:56.115876 18080 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: loading cluster ID for follower catalog manager: success
14:10:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:56.152136 18080 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: acquiring CA information for follower catalog manager: success
14:10:56.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:56.187000 18080 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 2e14717bfe4c4caea989247601d1934c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:56.777806 18085 debug-util.cc:398] Leaking SignalData structure 0x7b0800000a80 after lost signal to thread 18061
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:57.790657 18085 kernel_stack_watchdog.cc:198] Thread 18061 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:641 for 371ms:
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:57.791168 18089 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2397 milliseconds
14:10:57.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:55.396894 18087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:57.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:57.793385 18061 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 2.389s user 0.001s sys 0.000s
14:10:57.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:57.793639 18061 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 2.389s user 0.001s sys 0.000s
14:10:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:57.798683 18092 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.798777 18061 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:10:57.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.800060 18061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.802501 18061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.803898 18061 hybrid_clock.cc:648] HybridClock initialized: now 1751119857803859 us; error 46 us; skew 500 ppm
14:10:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.804746 18061 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.811829 18061 webserver.cc:469] Webserver started at http://127.17.92.129:43929/ using document root <none> and password file <none>
14:10:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.812695 18061 fs_manager.cc:362] Metadata directory not provided
14:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.812880 18061 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.813330 18061 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.817719 18061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data/instance:
14:10:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ade80e21244e1db781c71ce77b3438"
14:10:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:57 on dist-test-slave-n2gf"
14:10:57.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.818756 18061 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal/instance:
14:10:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ade80e21244e1db781c71ce77b3438"
14:10:57.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:57 on dist-test-slave-n2gf"
14:10:57.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.826207 18061 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
14:10:57.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.832996 18099 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:57.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.834415 18061 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.002s
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.834795 18061 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33ade80e21244e1db781c71ce77b3438"
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:57 on dist-test-slave-n2gf"
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.835163 18061 fs_report.cc:389] FS layout report
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal
14:10:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data/data
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:57.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.890605 18061 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.892078 18061 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.892482 18061 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:57.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.895774 18061 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.900125 18061 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.900321 18061 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:10:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.900517 18061 ts_tablet_manager.cc:610] Registered 0 tablets
14:10:57.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:57.900646 18061 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:10:58.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.070632 18061 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.129:39149
14:10:58.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.070786 18211 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.129:39149 every 8 connection(s)
14:10:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.073285 18061 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/data/info.pb
14:10:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.082870 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18061
14:10:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.083460 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-0/wal/instance
14:10:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.096803 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:10:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal
14:10:58.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/logs
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data/info.pb
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.130:0
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.130
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.130
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:58.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:10:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.114954 18213 heartbeater.cc:344] Connected to a master server at 127.17.92.188:40655
14:10:58.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.115504 18213 heartbeater.cc:461] Registering TS with master...
14:10:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.116838 18213 heartbeater.cc:507] Master 127.17.92.188:40655 requested a full tablet report, sending...
14:10:58.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.121531 17977 ts_manager.cc:194] Registered new tserver with Master: 33ade80e21244e1db781c71ce77b3438 (127.17.92.129:39149)
14:10:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.124913 18212 heartbeater.cc:344] Connected to a master server at 127.17.92.189:33237
14:10:58.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.125420 18212 heartbeater.cc:461] Registering TS with master...
14:10:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.126500 18212 heartbeater.cc:507] Master 127.17.92.189:33237 requested a full tablet report, sending...
14:10:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.129509 17898 ts_manager.cc:194] Registered new tserver with Master: 33ade80e21244e1db781c71ce77b3438 (127.17.92.129:39149)
14:10:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.130465 18214 heartbeater.cc:344] Connected to a master server at 127.17.92.190:38563
14:10:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.130811 18214 heartbeater.cc:461] Registering TS with master...
14:10:58.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.131834 18214 heartbeater.cc:507] Master 127.17.92.190:38563 requested a full tablet report, sending...
14:10:58.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.133867 17823 ts_manager.cc:194] Registered new tserver with Master: 33ade80e21244e1db781c71ce77b3438 (127.17.92.129:39149)
14:10:58.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.135766 17823 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.92.129:51997
14:10:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.450655 18222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:10:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.451186 18222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:10:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.451640 18222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:10:58.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.483239 18222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:10:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.484089 18222 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.92.130
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.518643 18222 tablet_server_runner.cc:78] Tablet server non-default flags:
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:10:58.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.130:0
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data/info.pb
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.130
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.130
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/logs
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:10:58.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:10:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:10:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:10:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:10:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.519965 18222 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:10:58.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.521512 18222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:10:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.539608 18229 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.138751 18214 heartbeater.cc:499] Master 127.17.92.190:38563 was elected leader, sending a full tablet report...
14:10:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.541630 18231 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:10:58.540767 18228 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:10:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:58.541489 18222 server_base.cc:1048] running on GCE node
14:10:59.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.760758 18222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:10:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.763195 18222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:10:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.764654 18222 hybrid_clock.cc:648] HybridClock initialized: now 1751119859764623 us; error 43 us; skew 500 ppm
14:10:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.765465 18222 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:10:59.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.772193 18222 webserver.cc:469] Webserver started at http://127.17.92.130:36051/ using document root <none> and password file <none>
14:10:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.773182 18222 fs_manager.cc:362] Metadata directory not provided
14:10:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.773408 18222 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:10:59.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.773840 18222 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:10:59.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.778267 18222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data/instance:
14:10:59.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1473f94306443a6b1288ada7079fe54"
14:10:59.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:59 on dist-test-slave-n2gf"
14:10:59.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.779312 18222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal/instance:
14:10:59.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1473f94306443a6b1288ada7079fe54"
14:10:59.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:59 on dist-test-slave-n2gf"
14:10:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.786741 18222 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
14:10:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.792519 18240 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:10:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.793681 18222 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:10:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.793967 18222 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal
14:10:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d1473f94306443a6b1288ada7079fe54"
14:10:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:10:59 on dist-test-slave-n2gf"
14:10:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.794291 18222 fs_report.cc:389] FS layout report
14:10:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data/data
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:10:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:10:59.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.861308 18222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:10:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.862812 18222 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:10:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.863238 18222 kserver.cc:163] Server-wide thread pool size limit: 3276
14:10:59.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.866207 18222 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:10:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.870852 18222 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:10:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.871088 18222 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:10:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.871287 18222 ts_tablet_manager.cc:610] Registered 0 tablets
14:10:59.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:10:59.871423 18222 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:11:00.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.010224 18222 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.130:35339
14:11:00.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.010313 18352 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.130:35339 every 8 connection(s)
14:11:00.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.012769 18222 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/data/info.pb
14:11:00.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.020474 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18222
14:11:00.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.021430 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-1/wal/instance
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.036959 17778 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/logs
14:11:00.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data/info.pb
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.131:0
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.131
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.131
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:00.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:11:00.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.059419 18354 heartbeater.cc:344] Connected to a master server at 127.17.92.188:40655
14:11:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.060031 18354 heartbeater.cc:461] Registering TS with master...
14:11:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.061807 18354 heartbeater.cc:507] Master 127.17.92.188:40655 requested a full tablet report, sending...
14:11:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.062111 18355 heartbeater.cc:344] Connected to a master server at 127.17.92.190:38563
14:11:00.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.062428 18355 heartbeater.cc:461] Registering TS with master...
14:11:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.063376 18355 heartbeater.cc:507] Master 127.17.92.190:38563 requested a full tablet report, sending...
14:11:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.064715 18353 heartbeater.cc:344] Connected to a master server at 127.17.92.189:33237
14:11:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.065047 18353 heartbeater.cc:461] Registering TS with master...
14:11:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.065606 17823 ts_manager.cc:194] Registered new tserver with Master: d1473f94306443a6b1288ada7079fe54 (127.17.92.130:35339)
14:11:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.065879 18353 heartbeater.cc:507] Master 127.17.92.189:33237 requested a full tablet report, sending...
14:11:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.065907 17977 ts_manager.cc:194] Registered new tserver with Master: d1473f94306443a6b1288ada7079fe54 (127.17.92.130:35339)
14:11:00.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.067497 17823 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.92.130:55487
14:11:00.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.068454 17898 ts_manager.cc:194] Registered new tserver with Master: d1473f94306443a6b1288ada7079fe54 (127.17.92.130:35339)
14:11:00.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.356489 18363 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.357016 18363 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:00.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.357514 18363 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:00.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.391247 18363 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:00.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.392108 18363 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.92.131
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.427740 18363 tablet_server_runner.cc:78] Tablet server non-default flags:
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.92.148:37099
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.92.131:0
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data/info.pb
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.92.131
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.92.190:38563,127.17.92.189:33237,127.17.92.188:40655
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.92.131
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/logs
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:00.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:00.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.429057 18363 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:00.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:00.430732 18363 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.446679 18371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.071414 18355 heartbeater.cc:499] Master 127.17.92.190:38563 was elected leader, sending a full tablet report...
14:11:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.446763 18370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:00.448556 18373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:01.724315 18372 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1275 milliseconds
14:11:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.724426 18363 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:01.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.725710 18363 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:01.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.728839 18363 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:01.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.730391 18363 hybrid_clock.cc:648] HybridClock initialized: now 1751119861730330 us; error 60 us; skew 500 ppm
14:11:01.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.731515 18363 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:01.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.737888 18363 webserver.cc:469] Webserver started at http://127.17.92.131:35935/ using document root <none> and password file <none>
14:11:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.738819 18363 fs_manager.cc:362] Metadata directory not provided
14:11:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.739056 18363 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:01.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.739542 18363 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:11:01.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.744516 18363 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data/instance:
14:11:01.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de94768b4c924d22911269e9f0478b34"
14:11:01.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:01 on dist-test-slave-n2gf"
14:11:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.745882 18363 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal/instance:
14:11:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de94768b4c924d22911269e9f0478b34"
14:11:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:01 on dist-test-slave-n2gf"
14:11:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.753126 18363 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
14:11:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.758666 18381 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.759683 18363 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:11:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.760046 18363 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal
14:11:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "de94768b4c924d22911269e9f0478b34"
14:11:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:01 on dist-test-slave-n2gf"
14:11:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.760443 18363 fs_report.cc:389] FS layout report
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data/data
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:01.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.812474 18363 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.813987 18363 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.814493 18363 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.817078 18363 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:11:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.821178 18363 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:11:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.821388 18363 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:11:01.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.821681 18363 ts_tablet_manager.cc:610] Registered 0 tablets
14:11:01.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.821851 18363 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:11:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.961047 18363 rpc_server.cc:307] RPC server started. Bound to: 127.17.92.131:43785
14:11:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.961165 18493 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.17.92.131:43785 every 8 connection(s)
14:11:01.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.963882 18363 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/data/info.pb
14:11:01.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.967864 17778 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18363
14:11:01.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:01.968358 17778 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster2801233951021693138/ts-2/wal/instance
14:11:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.006349 18494 heartbeater.cc:344] Connected to a master server at 127.17.92.189:33237
14:11:02.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.006817 18494 heartbeater.cc:461] Registering TS with master...
14:11:02.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.008140 18494 heartbeater.cc:507] Master 127.17.92.189:33237 requested a full tablet report, sending...
14:11:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.010406 18495 heartbeater.cc:344] Connected to a master server at 127.17.92.188:40655
14:11:02.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.010761 18495 heartbeater.cc:461] Registering TS with master...
14:11:02.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.011425 17898 ts_manager.cc:194] Registered new tserver with Master: de94768b4c924d22911269e9f0478b34 (127.17.92.131:43785)
14:11:02.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.011761 18495 heartbeater.cc:507] Master 127.17.92.188:40655 requested a full tablet report, sending...
14:11:02.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.014145 17977 ts_manager.cc:194] Registered new tserver with Master: de94768b4c924d22911269e9f0478b34 (127.17.92.131:43785)
14:11:02.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.017027 18496 heartbeater.cc:344] Connected to a master server at 127.17.92.190:38563
14:11:02.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.017319 18496 heartbeater.cc:461] Registering TS with master...
14:11:02.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.018260 18496 heartbeater.cc:507] Master 127.17.92.190:38563 requested a full tablet report, sending...
14:11:02.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.020562 17823 ts_manager.cc:194] Registered new tserver with Master: de94768b4c924d22911269e9f0478b34 (127.17.92.131:43785)
14:11:02.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.022192 17823 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.92.131:52843
14:11:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.026489 17778 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
14:11:02.037 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:11:02.037 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:11:02.038 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.92.190"
port: 38563
}
bound_http_address {
host: "127.17.92.190"
port: 36373
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.92.189"
port: 33237
}
bound_http_address {
host: "127.17.92.189"
port: 41525
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.92.188"
port: 40655
}
bound_http_address {
host: "127.17.92.188"
port: 45725
}
}
}
14:11:02.038 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:11:02.039 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.92.129"
port: 39149
}
bound_http_address {
host: "127.17.92.129"
port: 43929
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.92.130"
port: 35339
}
bound_http_address {
host: "127.17.92.130"
port: 36051
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.92.131"
port: 43785
}
bound_http_address {
host: "127.17.92.131"
port: 35935
}
}
}
14:11:02.039 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:11:02.041 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster6996158061126587646/krb5kdc/krb5cc
14:11:02.041 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:11:02.042 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGY_x37/test-tmp
14:11:02.042 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:11:02.042 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGY_x37/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:11:02.044 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:11:02.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:02.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:11:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.373624 18505 mini_oidc.cc:150] Starting JWKS server
14:11:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.377029 18505 webserver.cc:469] Webserver started at http://127.0.0.1:46077/ using document root <none> and password file <none>
14:11:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.377453 18505 mini_oidc.cc:186] Starting OIDC Discovery server
14:11:02.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:02.379184 18505 webserver.cc:469] Webserver started at http://127.0.0.1:42891/ using document root <none> and password file <none>
14:11:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:11:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/principal' for realm 'KRBTEST.COM',
14:11:02.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:11:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:02 dist-test-slave-n2gf krb5kdc[18516](info): setting up network...
14:11:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:11:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:02 dist-test-slave-n2gf krb5kdc[18516](info): set up 2 sockets
14:11:02.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:02 dist-test-slave-n2gf krb5kdc[18516](info): commencing operation
14:11:02.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:11:03.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:03.025833 18496 heartbeater.cc:499] Master 127.17.92.190:38563 was elected leader, sending a full tablet report...
14:11:04.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:04.504312 18505 mini_kdc.cc:121] Time spent starting KDC: real 2.086s user 0.008s sys 0.012s
14:11:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:11:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:11:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:11:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:11:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:11:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:11:04.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:11:04.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:11:04.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:11:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:11:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/test-user.keytab.
14:11:04.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/test-user.keytab.
14:11:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:04 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:04.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:04.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:11:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:11:04.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:11:04Z Disabled control of system clock
14:11:04.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.18.126@KRBTEST.COM; defaulting to no policy
14:11:04.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:04.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.18.126@KRBTEST.COM" created.
14:11:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab.
14:11:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab.
14:11:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:04.694511 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/logs
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.126:43773
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.126
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.126
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:04.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.006747 18531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:05.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.007408 18531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:05.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.008139 18531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.042999 18531 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.043284 18531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:05.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.043668 18531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.044003 18531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.044291 18531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.044570 18531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.044778 18531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:05.090173 18531 master_runner.cc:386] Master server non-default flags:
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:05.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.126
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.126:43773
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.126
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/logs
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:05.091943 18531 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:05.093070 18531 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.109182 18538 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.109230 18540 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:06.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:05.109207 18537 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:06.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.318107 18539 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:11:06.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.318248 18531 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:11:06.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.322086 18531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:06.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.324611 18531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:06.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.325932 18531 hybrid_clock.cc:648] HybridClock initialized: now 1751119866325904 us; error 48 us; skew 500 ppm
14:11:06.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.326699 18531 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:06.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:06 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119866, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:06.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.335884 18531 init.cc:377] Logged in from keytab as kudu/127.18.18.126@KRBTEST.COM (short username kudu)
14:11:06.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.340570 18531 webserver.cc:469] Webserver started at http://127.18.18.126:45549/ using document root <none> and password file <none>
14:11:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.341451 18531 fs_manager.cc:362] Metadata directory not provided
14:11:06.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.341635 18531 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:06.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.342015 18531 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:11:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.346236 18531 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/instance:
14:11:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcd902b4def4903b56ed3d93f8e1a00"
14:11:06.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:06 on dist-test-slave-n2gf"
14:11:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.347224 18531 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal/instance:
14:11:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcd902b4def4903b56ed3d93f8e1a00"
14:11:06.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:06 on dist-test-slave-n2gf"
14:11:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.354364 18531 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
14:11:06.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.360064 18549 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.361089 18531 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
14:11:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.361372 18531 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcd902b4def4903b56ed3d93f8e1a00"
14:11:06.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:06 on dist-test-slave-n2gf"
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.361665 18531 fs_report.cc:389] FS layout report
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/data
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.416329 18531 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:06.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.421756 18531 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:06.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.422144 18531 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.492805 18531 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.126:43773
14:11:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.492882 18600 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.126:43773 every 8 connection(s)
14:11:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.495599 18531 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.499208 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18531
14:11:06.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.499764 18505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal/instance
14:11:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.501667 18601 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:11:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.515355 18601 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } has no permanent_uuid. Determining permanent_uuid...
14:11:06.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.18.125@KRBTEST.COM; defaulting to no policy
14:11:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.18.125@KRBTEST.COM" created.
14:11:06.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:06 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119866, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab.
14:11:06.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab.
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.548884 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/logs
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.125:33259
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.125
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.125
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:06.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:06.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.555200 18601 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } has no permanent_uuid. Determining permanent_uuid...
14:11:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.557385 18552 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.561357 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.610785 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 1
14:11:06.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.614462 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.682801 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 2
14:11:06.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.686700 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.778070 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 3
14:11:06.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.781857 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.852754 18607 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:06.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.853294 18607 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.853927 18607 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.885115 18607 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.885401 18607 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:06.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.885699 18607 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.885907 18607 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.886117 18607 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.886346 18607 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:06.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.886494 18607 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.924484 18607 master_runner.cc:386] Master server non-default flags:
14:11:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.125
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.125:33259
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.125
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/logs
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.925249 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 4
14:11:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.925757 18607 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:06.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.927037 18607 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:06.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.929189 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:06.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.942361 18615 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:07.228633 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 5
14:11:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:07.233446 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:07.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:07.780793 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 6
14:11:07.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:07.787680 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:08.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.942613 18617 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:08.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:06.942713 18614 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:08.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:06.943974 18607 server_base.cc:1048] running on GCE node
14:11:08.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.158727 18607 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.162096 18607 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:08.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.163583 18607 hybrid_clock.cc:648] HybridClock initialized: now 1751119868163540 us; error 48 us; skew 500 ppm
14:11:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.164528 18607 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:08.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:08 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119868, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:08.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.171463 18607 init.cc:377] Logged in from keytab as kudu/127.18.18.125@KRBTEST.COM (short username kudu)
14:11:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.178076 18607 webserver.cc:469] Webserver started at http://127.18.18.125:41185/ using document root <none> and password file <none>
14:11:08.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.179037 18607 fs_manager.cc:362] Metadata directory not provided
14:11:08.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.179270 18607 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:08.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.179793 18607 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:11:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.184425 18607 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/instance:
14:11:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:08 on dist-test-slave-n2gf"
14:11:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.185621 18607 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal/instance:
14:11:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:08 on dist-test-slave-n2gf"
14:11:08.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.193161 18607 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
14:11:08.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.199440 18627 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:08.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.200551 18607 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:11:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.200891 18607 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:08 on dist-test-slave-n2gf"
14:11:08.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.201252 18607 fs_report.cc:389] FS layout report
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/data
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:08.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.273519 18607 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:08.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.279476 18607 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:08.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.280054 18607 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.354262 18607 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.125:33259
14:11:08.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.354326 18678 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.125:33259 every 8 connection(s)
14:11:08.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.357331 18607 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.362813 18679 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:11:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.366930 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18607
14:11:08.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.367412 18505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal/instance
14:11:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.377797 18679 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } has no permanent_uuid. Determining permanent_uuid...
14:11:08.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.18.124@KRBTEST.COM; defaulting to no policy
14:11:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.18.124@KRBTEST.COM" created.
14:11:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:08 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119868, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:08.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.411813 18679 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } has no permanent_uuid. Determining permanent_uuid...
14:11:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:11:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab.
14:11:08.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.18.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab.
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.417513 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/logs
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:08.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:08.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.124:36807
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.124
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.124
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:08.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:08.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:08 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119868, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:08.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.445148 18679 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } has no permanent_uuid. Determining permanent_uuid...
14:11:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.447324 18630 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.451539 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.501197 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 1
14:11:08.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.506047 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.574561 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 2
14:11:08.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.579180 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.670662 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 3
14:11:08.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.674547 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.740046 18687 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:08.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.740600 18687 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:08.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.741214 18687 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:08.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.773885 18687 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:08.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.774134 18687 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:08.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.774400 18687 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:08.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.774688 18687 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.774901 18687 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.775087 18687 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:08.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.775242 18687 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.813612 18687 master_runner.cc:386] Master server non-default flags:
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.124
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.124:36807
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.124
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/logs
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:08.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.814944 18687 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:08.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.816113 18687 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.817894 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 4
14:11:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.821571 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.833113 18696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:08.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.848306 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } attempt: 7
14:11:08.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:08 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119866, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:08.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.872113 18601 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } has no permanent_uuid. Determining permanent_uuid...
14:11:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.878199 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:08.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.936829 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 1
14:11:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.941857 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.023301 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 2
14:11:09.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.027730 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.113124 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 3
14:11:09.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.118660 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.120929 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 5
14:11:09.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.124858 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.259160 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 4
14:11:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.264926 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.548295 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 5
14:11:09.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.553427 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:09.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:09.672442 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 6
14:11:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:09.679209 18679 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:10.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:08.835428 18687 server_base.cc:1048] running on GCE node
14:11:10.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.833300 18695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:10.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:08.834767 18698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.071736 18687 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.074668 18687 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:10.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.076042 18687 hybrid_clock.cc:648] HybridClock initialized: now 1751119870076006 us; error 48 us; skew 500 ppm
14:11:10.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.076893 18687 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:10.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.083093 18687 init.cc:377] Logged in from keytab as kudu/127.18.18.124@KRBTEST.COM (short username kudu)
14:11:10.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.088609 18687 webserver.cc:469] Webserver started at http://127.18.18.124:38013/ using document root <none> and password file <none>
14:11:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.089515 18687 fs_manager.cc:362] Metadata directory not provided
14:11:10.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.089706 18687 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:10.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.090109 18687 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
14:11:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.094709 18687 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/instance:
14:11:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11f46b1623db4bbeb64a0c50fceef758"
14:11:10.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:10 on dist-test-slave-n2gf"
14:11:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.095908 18687 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal/instance:
14:11:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11f46b1623db4bbeb64a0c50fceef758"
14:11:10.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:10 on dist-test-slave-n2gf"
14:11:10.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.103513 18687 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
14:11:10.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.105824 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 6
14:11:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.113502 18712 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.114773 18601 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.115218 18687 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.006s sys 0.003s
14:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.115538 18687 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11f46b1623db4bbeb64a0c50fceef758"
14:11:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:10 on dist-test-slave-n2gf"
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.115897 18687 fs_report.cc:389] FS layout report
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/data
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:10.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:10.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.192987 18687 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.199187 18687 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.199738 18687 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.271028 18687 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.124:36807
14:11:10.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.271157 18764 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.124:36807 every 8 connection(s)
14:11:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.273787 18687 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:10.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.274672 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18687
14:11:10.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.275195 18505 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal/instance
14:11:10.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.279873 18765 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:11:10.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.292019 18765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } has no permanent_uuid. Determining permanent_uuid...
14:11:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:10.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:10.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.323612 18765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } has no permanent_uuid. Determining permanent_uuid...
14:11:10.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:10.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:10.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.352312 18765 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } has no permanent_uuid. Determining permanent_uuid...
14:11:10.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:10.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119864, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:10.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.384595 18505 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
14:11:10.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.385437 18765 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Bootstrap starting.
14:11:10.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.391600 18765 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Neither blocks nor log segments found. Creating new log.
14:11:10.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.393353 18765 log.cc:826] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Log is configured to *not* fsync() on all Append() calls
14:11:10.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:10.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:11:10.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:11:10.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:11:10.396 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.18.126"
port: 43773
}
bound_http_address {
host: "127.18.18.126"
port: 45549
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.18.125"
port: 33259
}
bound_http_address {
host: "127.18.18.125"
port: 41185
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.18.124"
port: 36807
}
bound_http_address {
host: "127.18.18.124"
port: 38013
}
}
}
14:11:10.397 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:11:10.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:10.398 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:11:10.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:11:10.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.399065 18765 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: No bootstrap required, opened a new log
14:11:10.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.422504 18765 raft_consensus.cc:357] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.423179 18765 raft_consensus.cc:383] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:10.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.423475 18765 raft_consensus.cc:738] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11f46b1623db4bbeb64a0c50fceef758, State: Initialized, Role: FOLLOWER
14:11:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.424252 18765 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.427230 18777 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.428067 18777 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: FOLLOWER
14:11:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.429095 18765 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: configured and running, proceeding with master startup.
14:11:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:10.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:11:10.436 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:10.437 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc
14:11:10.438 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:11:10.438 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-user
14:11:10.439 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-user"
}
14:11:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.454766 18719 tablet.cc:2378] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:11:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.459328 18789 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.459653 18789 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-user@KRBTEST.COM:
14:11:10.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.455291 18687 master_runner.cc:186] Error getting master registration for 127.18.18.126:43773: OK, instance_id { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" instance_seqno: 1751119866473255 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:11:10.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.484321 18687 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:10.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.499898 18777 raft_consensus.cc:491] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.500504 18777 raft_consensus.cc:513] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.503038 18777 leader_election.cc:290] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4bcd902b4def4903b56ed3d93f8e1a00 (127.18.18.126:43773), ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259)
14:11:10.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.739894 18679 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 7
14:11:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:10 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119868, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:10.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.776301 18679 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Bootstrap starting.
14:11:10.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.783987 18679 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Neither blocks nor log segments found. Creating new log.
14:11:10.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.786329 18679 log.cc:826] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Log is configured to *not* fsync() on all Append() calls
14:11:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.792452 18679 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: No bootstrap required, opened a new log
14:11:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.820209 18679 raft_consensus.cc:357] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.821225 18679 raft_consensus.cc:383] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.821568 18679 raft_consensus.cc:738] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ecb4c0a9906c4f6a85bff6194ca77ba4, State: Initialized, Role: FOLLOWER
14:11:10.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.822525 18679 consensus_queue.cc:260] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.825754 18794 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:10.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.826640 18794 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.827857 18679 sys_catalog.cc:564] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: configured and running, proceeding with master startup.
14:11:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.828022 18654 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11f46b1623db4bbeb64a0c50fceef758" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" is_pre_election: true
14:11:10.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.828912 18654 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 11f46b1623db4bbeb64a0c50fceef758 in term 0.
14:11:10.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.830368 18716 leader_election.cc:304] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11f46b1623db4bbeb64a0c50fceef758, ecb4c0a9906c4f6a85bff6194ca77ba4; no voters:
14:11:10.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.831290 18777 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:11:10.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.831784 18777 raft_consensus.cc:491] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:10.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.832119 18777 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 0 FOLLOWER]: Advancing to term 1
14:11:10.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.840127 18777 raft_consensus.cc:513] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.842761 18777 leader_election.cc:290] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [CANDIDATE]: Term 1 election: Requested vote from peers 4bcd902b4def4903b56ed3d93f8e1a00 (127.18.18.126:43773), ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259)
14:11:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.843353 18654 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11f46b1623db4bbeb64a0c50fceef758" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.843951 18654 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 0 FOLLOWER]: Advancing to term 1
14:11:10.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.851670 18654 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 11f46b1623db4bbeb64a0c50fceef758 in term 1.
14:11:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.853248 18716 leader_election.cc:304] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 11f46b1623db4bbeb64a0c50fceef758, ecb4c0a9906c4f6a85bff6194ca77ba4; no voters:
14:11:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.854246 18777 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 FOLLOWER]: Leader election won for term 1
14:11:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.856405 18777 raft_consensus.cc:695] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 LEADER]: Becoming Leader. State: Replica: 11f46b1623db4bbeb64a0c50fceef758, State: Running, Role: LEADER
14:11:10.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.857443 18777 consensus_queue.cc:237] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:10.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.876395 18807 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:11:10.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:10.876993 18807 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:11:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.880707 18805 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11f46b1623db4bbeb64a0c50fceef758. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:10.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.881517 18805 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: LEADER
14:11:10.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.889657 18809 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:11:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.905546 18809 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:11:10.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.874733 18607 master_runner.cc:186] Error getting master registration for 127.18.18.126:43773: OK, instance_id { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" instance_seqno: 1751119866473255 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:11:10.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.913115 18607 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:10.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.935273 18654 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Refusing update from remote peer 11f46b1623db4bbeb64a0c50fceef758: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:11:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.937353 18805 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:11:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.974309 18794 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11f46b1623db4bbeb64a0c50fceef758. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.975354 18794 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:10.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.986150 18777 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:10.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.986903 18777 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: LEADER
14:11:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.995818 18809 catalog_manager.cc:1349] Generated new cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.996148 18809 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:11:11.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.997097 18794 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:11.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:10.997790 18794 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.036501 18809 catalog_manager.cc:1372] Generated new certificate authority record
14:11:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.038584 18809 catalog_manager.cc:1506] Loading token signing keys...
14:11:11.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.080526 18809 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Generated new TSK 0
14:11:11.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.081715 18809 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:11:11.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.148204 18601 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } attempt: 7
14:11:11.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:11 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119866, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.219983 18601 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Bootstrap starting.
14:11:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.226904 18601 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Neither blocks nor log segments found. Creating new log.
14:11:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.229369 18601 log.cc:826] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Log is configured to *not* fsync() on all Append() calls
14:11:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.241199 18601 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: No bootstrap required, opened a new log
14:11:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.270269 18601 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.271405 18601 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:11:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.272341 18601 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4bcd902b4def4903b56ed3d93f8e1a00, State: Initialized, Role: FOLLOWER
14:11:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.273825 18601 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.302330 18601 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: configured and running, proceeding with master startup.
14:11:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.302492 18576 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11f46b1623db4bbeb64a0c50fceef758" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" is_pre_election: true
14:11:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.303342 18576 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 11f46b1623db4bbeb64a0c50fceef758 in term 0.
14:11:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.306941 18574 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 0 FOLLOWER]: Advancing to term 1
14:11:11.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.303931 18575 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "11f46b1623db4bbeb64a0c50fceef758" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4bcd902b4def4903b56ed3d93f8e1a00"
14:11:11.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.321066 18574 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Refusing update from remote peer 11f46b1623db4bbeb64a0c50fceef758: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:11:11.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.301270 18818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:11.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.328162 18818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: FOLLOWER
14:11:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.343313 18805 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:11:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.441232 18826 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:11.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.443984 18826 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: FOLLOWER
14:11:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.444825 18818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 11f46b1623db4bbeb64a0c50fceef758. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.445859 18818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: FOLLOWER
14:11:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.512614 18834 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.741708 18805 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "11f46b1623db4bbeb64a0c50fceef758" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.742436 18805 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: LEADER
14:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.884725 18807 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.885020 18807 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: loading cluster ID for follower catalog manager: success
14:11:11.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.892287 18807 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: acquiring CA information for follower catalog manager: success
14:11:11.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:11.898231 18807 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:12.529655 18834 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:12.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:12.530493 18834 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: loading cluster ID for follower catalog manager: success
14:11:12.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:12.538647 18834 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: acquiring CA information for follower catalog manager: success
14:11:12.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:12.545575 18834 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:13.482 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.18.126:43773
14:11:13.483 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:11:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.485014 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18531
14:11:13.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.511000 18716 connection.cc:537] client connection to 127.18.18.126:43773 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:11:13.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:13.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.511508 18716 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:11:13.513 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.18.125:33259
14:11:13.513 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:11:13.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.513885 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18607
14:11:13.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.516916 18716 consensus_peers.cc:489] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 -> Peer 4bcd902b4def4903b56ed3d93f8e1a00 (127.18.18.126:43773): Couldn't send request to peer 4bcd902b4def4903b56ed3d93f8e1a00. Status: Network error: Client connection negotiation failed: client connection to 127.18.18.126:43773: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:13.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:13.544 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.18.124:36807
14:11:13.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:11:13.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.545287 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18687
14:11:13.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:13.573 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.18.126:43773
14:11:13.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.575536 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/logs
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.126:43773
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.126
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45549
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.126
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:13.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:13.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.885174 18845 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:13.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.885730 18845 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.886382 18845 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.918525 18845 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:13.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.918825 18845 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.919195 18845 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:13.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.919508 18845 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.919832 18845 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.920141 18845 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.920369 18845 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.958165 18845 master_runner.cc:386] Master server non-default flags:
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.126.keytab
14:11:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.126
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.126:43773
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.126
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45549
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/logs
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.959893 18845 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.961079 18845 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:13.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.977895 18851 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:15.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.977941 18852 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:13.980224 18845 server_base.cc:1048] running on GCE node
14:11:15.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:13.978935 18854 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.194782 18845 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.197539 18845 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.198921 18845 hybrid_clock.cc:648] HybridClock initialized: now 1751119875198876 us; error 49 us; skew 500 ppm
14:11:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.199944 18845 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:15 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119875, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:15.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.210800 18845 init.cc:377] Logged in from keytab as kudu/127.18.18.126@KRBTEST.COM (short username kudu)
14:11:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.216162 18845 webserver.cc:469] Webserver started at http://127.18.18.126:45549/ using document root <none> and password file <none>
14:11:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.217033 18845 fs_manager.cc:362] Metadata directory not provided
14:11:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.217238 18845 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.224956 18845 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
14:11:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.229573 18862 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.230674 18845 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.231030 18845 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bcd902b4def4903b56ed3d93f8e1a00"
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:06 on dist-test-slave-n2gf"
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.233052 18845 fs_report.cc:389] FS layout report
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/wal
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/data
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:15.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.297808 18845 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.304010 18845 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:15.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.304504 18845 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:15.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.379104 18845 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.126:43773
14:11:15.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.379208 18914 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.126:43773 every 8 connection(s)
14:11:15.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.382022 18845 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-0/data/info.pb
14:11:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.392102 18915 sys_catalog.cc:263] Verifying existing consensus state
14:11:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.392544 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18845
14:11:15.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:15.393 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.18.125:33259
14:11:15.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.395203 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/logs
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:15.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.125:33259
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.125
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41185
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.125
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.400851 18915 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Bootstrap starting.
14:11:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.441030 18915 log.cc:826] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Log is configured to *not* fsync() on all Append() calls
14:11:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.459614 18915 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.460616 18915 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Bootstrap complete.
14:11:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.482627 18915 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.485036 18915 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4bcd902b4def4903b56ed3d93f8e1a00, State: Initialized, Role: FOLLOWER
14:11:15.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.485960 18915 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:15.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.488878 18919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:15.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.489727 18919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: FOLLOWER
14:11:15.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.490672 18915 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: configured and running, proceeding with master startup.
14:11:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.520900 18930 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:15.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.521200 18930 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: loading cluster ID for follower catalog manager: success
14:11:15.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.529090 18930 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: acquiring CA information for follower catalog manager: success
14:11:15.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.534919 18930 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:15.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:15 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119875, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.573236 18845 master_runner.cc:186] Error getting master registration for 127.18.18.125:33259: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111),
14:11:15.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.602221 18845 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.762010 18917 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.762557 18917 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:15.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.763280 18917 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.796061 18917 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.796319 18917 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.796578 18917 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:15.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.796770 18917 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.796957 18917 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.797134 18917 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:15.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.797294 18917 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.835400 18917 master_runner.cc:386] Master server non-default flags:
14:11:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:15.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.125.keytab
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.125
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.125:33259
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.125
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41185
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/logs
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:15.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.836736 18917 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.837731 18917 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:15.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.853931 18940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.025657 18945 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:17.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.026335 18945 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.029306 18945 leader_election.cc:290] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259), 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807)
14:11:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.033842 18866 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:17.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.038859 18866 leader_election.cc:336] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259): Network error: Client connection negotiation failed: client connection to 127.18.18.125:33259: connect: Connection refused (error 111)
14:11:17.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.039475 18866 leader_election.cc:336] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807): Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.039863 18866 leader_election.cc:304] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4bcd902b4def4903b56ed3d93f8e1a00; no voters: 11f46b1623db4bbeb64a0c50fceef758, ecb4c0a9906c4f6a85bff6194ca77ba4
14:11:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.040645 18945 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:11:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.857882 18942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:15.858284 18917 server_base.cc:1048] running on GCE node
14:11:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:15.857523 18939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:17.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.191560 18917 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.194995 18917 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.197181 18917 hybrid_clock.cc:648] HybridClock initialized: now 1751119877197152 us; error 29 us; skew 500 ppm
14:11:17.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.198407 18917 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:17 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119877, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.205562 18917 init.cc:377] Logged in from keytab as kudu/127.18.18.125@KRBTEST.COM (short username kudu)
14:11:17.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.212363 18917 webserver.cc:469] Webserver started at http://127.18.18.125:41185/ using document root <none> and password file <none>
14:11:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.213338 18917 fs_manager.cc:362] Metadata directory not provided
14:11:17.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.213575 18917 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.222198 18917 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.004s
14:11:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.227196 18954 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.228308 18917 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:11:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.228621 18917 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:08 on dist-test-slave-n2gf"
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.230661 18917 fs_report.cc:389] FS layout report
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/wal
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/data
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:17.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.305986 18917 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.311770 18917 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:17.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.312268 18917 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:17.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.406229 18917 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.125:33259
14:11:17.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.406364 19005 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.125:33259 every 8 connection(s)
14:11:17.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.409128 18917 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-1/data/info.pb
14:11:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.415105 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 18917
14:11:17.415 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:17.415 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.18.124:36807
14:11:17.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.417148 18505 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/logs
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:11:17.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.124:36807
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.124
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38013
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.124
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:17.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:11:17.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.423395 19006 sys_catalog.cc:263] Verifying existing consensus state
14:11:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.431411 19006 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Bootstrap starting.
14:11:17.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.485603 19006 log.cc:826] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Log is configured to *not* fsync() on all Append() calls
14:11:17.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.505491 19006 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:17.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.506490 19006 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: Bootstrap complete.
14:11:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.541045 19006 raft_consensus.cc:357] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:17.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.544657 19006 raft_consensus.cc:738] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ecb4c0a9906c4f6a85bff6194ca77ba4, State: Initialized, Role: FOLLOWER
14:11:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.545729 19006 consensus_queue.cc:260] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.548875 19010 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.549942 19010 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.550683 19006 sys_catalog.cc:564] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: configured and running, proceeding with master startup.
14:11:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.607642 19021 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:17.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.610293 19021 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: loading cluster ID for follower catalog manager: success
14:11:17.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.617620 19021 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: acquiring CA information for follower catalog manager: success
14:11:17.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.626092 19021 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:17.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:17 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119877, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.697202 18917 master_runner.cc:186] Error getting master registration for 127.18.18.124:36807: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111),
14:11:17.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.737270 18917 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:11:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.856930 19008 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:11:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.857474 19008 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:11:17.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.858065 19008 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:11:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.889920 19008 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:11:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.890225 19008 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:11:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.890511 19008 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:11:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.890712 19008 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:11:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.891022 19008 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:11:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.891345 19008 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:11:17.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.891566 19008 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46077/jwks.json
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.930941 19008 master_runner.cc:386] Master server non-default flags:
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.18.84:33923
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:11:17.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/kudu_127.18.18.124.keytab
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.18.124
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.18.124:36807
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46077/jwks.json
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.18.124
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38013
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/logs
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision f0776864a65c19ebdd909053f3e624e8bcc25cee
14:11:17.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:11:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 28 Jun 2025 13:43:22 UTC on 5fd53c4cbb9d
14:11:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6813
14:11:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:11:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.932286 19008 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:11:17.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.933311 19008 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:11:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.950002 19033 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:18.277424 17929 debug-util.cc:398] Leaking SignalData structure 0x7b080009f940 after lost signal to thread 17864
14:11:18.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:18.278142 17929 debug-util.cc:398] Leaking SignalData structure 0x7b08000935c0 after lost signal to thread 17932
14:11:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.864988 19040 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:11:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.866163 19040 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:18.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.878700 19040 leader_election.cc:290] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259), 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807)
14:11:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:18.898610 18866 leader_election.cc:336] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807): Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.904839 18981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" is_pre_election: true
14:11:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.906224 18981 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4bcd902b4def4903b56ed3d93f8e1a00 in term 1.
14:11:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.909035 18866 leader_election.cc:304] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4bcd902b4def4903b56ed3d93f8e1a00, ecb4c0a9906c4f6a85bff6194ca77ba4; no voters: 11f46b1623db4bbeb64a0c50fceef758
14:11:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.910817 19040 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:11:18.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.911319 19040 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:11:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.911813 19040 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 1 FOLLOWER]: Advancing to term 2
14:11:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.933331 19040 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.938261 18981 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4"
14:11:18.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.939257 18981 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 1 FOLLOWER]: Advancing to term 2
14:11:18.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.954950 19040 leader_election.cc:290] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 election: Requested vote from peers ecb4c0a9906c4f6a85bff6194ca77ba4 (127.18.18.125:33259), 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807)
14:11:18.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:18.957149 18866 leader_election.cc:336] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807): Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111)
14:11:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.996495 18981 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4bcd902b4def4903b56ed3d93f8e1a00 in term 2.
14:11:19.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:18.999184 18866 leader_election.cc:304] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4bcd902b4def4903b56ed3d93f8e1a00, ecb4c0a9906c4f6a85bff6194ca77ba4; no voters: 11f46b1623db4bbeb64a0c50fceef758
14:11:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.001305 19040 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 2 FOLLOWER]: Leader election won for term 2
14:11:19.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.013761 19040 raft_consensus.cc:695] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [term 2 LEADER]: Becoming Leader. State: Replica: 4bcd902b4def4903b56ed3d93f8e1a00, State: Running, Role: LEADER
14:11:19.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.015913 19040 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.039103 19045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4bcd902b4def4903b56ed3d93f8e1a00. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.040829 19045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: LEADER
14:11:19.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.049101 19047 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:11:19.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.060745 19047 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:11:19.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.067637 19047 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:19.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.068143 19047 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:11:19.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.092650 19047 catalog_manager.cc:1506] Loading token signing keys...
14:11:19.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.102128 19047 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00: Loaded TSK: 0
14:11:19.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.103690 19047 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:11:19.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:17.952205 19008 server_base.cc:1048] running on GCE node
14:11:19.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:17.950968 19032 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:19.339311 19035 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:11:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.343948 19008 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:11:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.347285 19008 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:11:19.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.349467 19008 hybrid_clock.cc:648] HybridClock initialized: now 1751119879349433 us; error 49 us; skew 500 ppm
14:11:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.350325 19008 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:11:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:19 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119879, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.357308 19008 init.cc:377] Logged in from keytab as kudu/127.18.18.124@KRBTEST.COM (short username kudu)
14:11:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.364638 19008 webserver.cc:469] Webserver started at http://127.18.18.124:38013/ using document root <none> and password file <none>
14:11:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.365706 19008 fs_manager.cc:362] Metadata directory not provided
14:11:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.365962 19008 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:11:19.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.375288 19008 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
14:11:19.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.399255 19054 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:11:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.411237 19008 fs_manager.cc:730] Time spent opening block manager: real 0.033s user 0.005s sys 0.013s
14:11:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.411933 19008 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data,/tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "11f46b1623db4bbeb64a0c50fceef758"
14:11:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-28 14:11:10 on dist-test-slave-n2gf"
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.415068 19008 fs_report.cc:389] FS layout report
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/wal
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/data
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:11:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:11:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:11:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.439865 18981 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [term 2 FOLLOWER]: Refusing update from remote peer 4bcd902b4def4903b56ed3d93f8e1a00: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:11:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.441864 19045 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:11:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.525833 19060 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:19.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.526748 19060 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.528327 19040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:19.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.529111 19040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: LEADER
14:11:19.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.511054 19055 sys_catalog.cc:455] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4bcd902b4def4903b56ed3d93f8e1a00. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:19.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.531298 19055 sys_catalog.cc:458] T 00000000000000000000000000000000 P ecb4c0a9906c4f6a85bff6194ca77ba4 [sys.catalog]: This master's current role is: FOLLOWER
14:11:19.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:19.547909 18866 consensus_peers.cc:489] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 -> Peer 11f46b1623db4bbeb64a0c50fceef758 (127.18.18.124:36807): Couldn't send request to peer 11f46b1623db4bbeb64a0c50fceef758. Status: Network error: Client connection negotiation failed: client connection to 127.18.18.124:36807: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:11:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.562042 19008 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:11:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.568691 19008 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:11:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.569331 19008 kserver.cc:163] Server-wide thread pool size limit: 3276
14:11:19.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.773721 19008 rpc_server.cc:307] RPC server started. Bound to: 127.18.18.124:36807
14:11:19.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.773793 19112 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.18.18.124:36807 every 8 connection(s)
14:11:19.778 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:11:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.777302 18505 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu as pid 19008
14:11:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.776799 19008 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/master-2/data/info.pb
14:11:19.779 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:19.788 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.18.125:33259(127.18.18.125:33259)] connecting to peer
14:11:19.792 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.18.124:36807(127.18.18.124:36807)] connecting to peer
14:11:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.798431 19114 sys_catalog.cc:263] Verifying existing consensus state
14:11:19.806 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:19.808 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.124:36807(127.18.18.124:36807)] Successfully connected to peer
14:11:19.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.817414 19114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Bootstrap starting.
14:11:19.828 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.125:33259(127.18.18.125:33259)] Successfully connected to peer
14:11:19.841 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:11:19.842 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc
14:11:19.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:19 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, test-user@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:19.862 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-user@KRBTEST.COM
14:11:19.862 [ERROR - kudu-nio-1] (SecurityContext.java:246) Attempted to refresh Kerberos credentials from ticket cache but found that the new Kerberos principal test-user@KRBTEST.COM did not match the original principal test-admin@KRBTEST.COM. Ignoring.
14:11:19.880 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.18.126
14:11:19.884 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.18.125
14:11:19.889 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.18.124
14:11:19.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.914978 19114 log.cc:826] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Log is configured to *not* fsync() on all Append() calls
14:11:19.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.947000 19114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:11:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:19 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:19.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.948385 19114 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: Bootstrap complete.
14:11:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:19 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119870, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.987447 19114 raft_consensus.cc:357] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.991113 19114 raft_consensus.cc:738] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 11f46b1623db4bbeb64a0c50fceef758, State: Initialized, Role: FOLLOWER
14:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:19.992328 19114 consensus_queue.cc:260] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } }
14:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.001080 19122 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.002075 19122 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: FOLLOWER
14:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.006089 19114 sys_catalog.cc:564] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: configured and running, proceeding with master startup.
14:11:20.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.088596 19133 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:11:20.133 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xb8f573bc, L:/127.0.0.1:45488 - R:/127.18.18.125:33259] using SASL/GSSAPI
14:11:20.135 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:11:20.135 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:11:20.136 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.139037 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.802119 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37400 (local address 127.18.18.126:43773)
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849190 (+ 47071us) server_negotiation.cc:206] Beginning negotiation
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849223 (+ 33us) server_negotiation.cc:399] Waiting for connection header
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849282 (+ 59us) server_negotiation.cc:407] Connection header received
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849449 (+ 167us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849484 (+ 35us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.849747 (+ 263us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.867805 (+ 18058us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.896030 (+ 28225us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:19.902582 (+ 6552us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.133205 (+230623us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.133935 (+ 730us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.137091 (+ 3156us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37400: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
14:11:20.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":46341,"thread_start_us":65346,"threads_started":1}
14:11:20.146 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x7119f722, L:/127.0.0.1:54784 - R:/127.18.18.124:36807] using SASL/GSSAPI
14:11:20.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:20 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119879, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.126@KRBTEST.COM
14:11:20.178 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.180 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:20 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119875, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:20.216 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.240 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.240 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:20 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119879, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.125@KRBTEST.COM
14:11:20.241 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.246929 19088 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [term 1 FOLLOWER]: Advancing to term 2
14:11:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.247393 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.202220 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37422 (local address 127.18.18.126:43773)
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.203169 (+ 949us) server_negotiation.cc:206] Beginning negotiation
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.203207 (+ 38us) server_negotiation.cc:399] Waiting for connection header
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.216976 (+ 13769us) server_negotiation.cc:407] Connection header received
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.223099 (+ 6123us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.223140 (+ 41us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.229770 (+ 6630us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.229966 (+ 196us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.246399 (+ 16433us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37422: BlockingRecv error: recv got EOF from 127.0.0.1:37422 (error 108)
14:11:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":317}
14:11:20.249 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.260 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.268 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.275 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.276 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.276 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.278280 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.264929 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37424 (local address 127.18.18.126:43773)
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.266349 (+ 1420us) server_negotiation.cc:206] Beginning negotiation
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.266385 (+ 36us) server_negotiation.cc:399] Waiting for connection header
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.269037 (+ 2652us) server_negotiation.cc:407] Connection header received
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.269233 (+ 196us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.269268 (+ 35us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.269535 (+ 267us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.269732 (+ 197us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.277172 (+ 7440us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37424: BlockingRecv error: recv got EOF from 127.0.0.1:37424 (error 108)
14:11:20.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":571}
14:11:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:20 dist-test-slave-n2gf krb5kdc[18516](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119879, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for kudu/127.18.18.124@KRBTEST.COM
14:11:20.324 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.324657 19122 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4bcd902b4def4903b56ed3d93f8e1a00. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:20.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.325587 19122 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: FOLLOWER
14:11:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.325230 19140 sys_catalog.cc:455] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:20.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.326195 19140 sys_catalog.cc:458] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758 [sys.catalog]: This master's current role is: FOLLOWER
14:11:20.340 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.343 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.346 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.354571 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.344033 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37428 (local address 127.18.18.126:43773)
14:11:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.344872 (+ 839us) server_negotiation.cc:206] Beginning negotiation
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.344904 (+ 32us) server_negotiation.cc:399] Waiting for connection header
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.344960 (+ 56us) server_negotiation.cc:407] Connection header received
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.345114 (+ 154us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.345147 (+ 33us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.345422 (+ 275us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.345626 (+ 204us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.353513 (+ 7887us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37428: BlockingRecv error: recv got EOF from 127.0.0.1:37428 (error 108)
14:11:20.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":153}
14:11:20.358 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.370 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.374 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.387 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.387 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.389089 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.373092 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37440 (local address 127.18.18.126:43773)
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.373863 (+ 771us) server_negotiation.cc:206] Beginning negotiation
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.373893 (+ 30us) server_negotiation.cc:399] Waiting for connection header
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.377487 (+ 3594us) server_negotiation.cc:407] Connection header received
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.377658 (+ 171us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.377690 (+ 32us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.377941 (+ 251us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.378138 (+ 197us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.387981 (+ 9843us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37440: BlockingRecv error: recv got EOF from 127.0.0.1:37440 (error 108)
14:11:20.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":98}
14:11:20.419 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.430 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.432 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.433 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.435263 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.427563 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37452 (local address 127.18.18.126:43773)
14:11:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.428376 (+ 813us) server_negotiation.cc:206] Beginning negotiation
14:11:20.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.428402 (+ 26us) server_negotiation.cc:399] Waiting for connection header
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.430506 (+ 2104us) server_negotiation.cc:407] Connection header received
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.430869 (+ 363us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.430903 (+ 34us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.431151 (+ 248us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.431359 (+ 208us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.434150 (+ 2791us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37452: BlockingRecv error: recv got EOF from 127.0.0.1:37452 (error 108)
14:11:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":89}
14:11:20.499 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.508 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.511 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.513036 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.505750 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37454 (local address 127.18.18.126:43773)
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.506549 (+ 799us) server_negotiation.cc:206] Beginning negotiation
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.506573 (+ 24us) server_negotiation.cc:399] Waiting for connection header
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.509260 (+ 2687us) server_negotiation.cc:407] Connection header received
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.509430 (+ 170us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.509459 (+ 29us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.509697 (+ 238us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.509864 (+ 167us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.512128 (+ 2264us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37454: BlockingRecv error: recv got EOF from 127.0.0.1:37454 (error 108)
14:11:20.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":227}
14:11:20.560 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.563 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.567 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.567 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.568 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.570034 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.562947 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37464 (local address 127.18.18.126:43773)
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.565124 (+ 2177us) server_negotiation.cc:206] Beginning negotiation
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.565148 (+ 24us) server_negotiation.cc:399] Waiting for connection header
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.565201 (+ 53us) server_negotiation.cc:407] Connection header received
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.565362 (+ 161us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.565390 (+ 28us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.566536 (+ 1146us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.566742 (+ 206us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.568964 (+ 2222us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37464: BlockingRecv error: recv got EOF from 127.0.0.1:37464 (error 108)
14:11:20.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1586}
14:11:20.570 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.671717 19045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4bcd902b4def4903b56ed3d93f8e1a00" member_type: VOTER last_known_addr { host: "127.18.18.126" port: 43773 } } peers { permanent_uuid: "ecb4c0a9906c4f6a85bff6194ca77ba4" member_type: VOTER last_known_addr { host: "127.18.18.125" port: 33259 } } peers { permanent_uuid: "11f46b1623db4bbeb64a0c50fceef758" member_type: VOTER last_known_addr { host: "127.18.18.124" port: 36807 } } }
14:11:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:20.672587 19045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4bcd902b4def4903b56ed3d93f8e1a00 [sys.catalog]: This master's current role is: LEADER
14:11:20.720 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.722 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.725 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.725 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.726 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.727690 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.721955 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37478 (local address 127.18.18.126:43773)
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724087 (+ 2132us) server_negotiation.cc:206] Beginning negotiation
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724121 (+ 34us) server_negotiation.cc:399] Waiting for connection header
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724176 (+ 55us) server_negotiation.cc:407] Connection header received
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724384 (+ 208us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724434 (+ 50us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.724702 (+ 268us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.725788 (+ 1086us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.726795 (+ 1007us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37478: BlockingRecv error: recv got EOF from 127.0.0.1:37478 (error 108)
14:11:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1458}
14:11:20.899 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:20.902 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:20.904 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.904 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:20.905 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:20.905 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:20.906497 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.902363 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37484 (local address 127.18.18.126:43773)
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903104 (+ 741us) server_negotiation.cc:206] Beginning negotiation
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903132 (+ 28us) server_negotiation.cc:399] Waiting for connection header
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903188 (+ 56us) server_negotiation.cc:407] Connection header received
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903327 (+ 139us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903352 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.903956 (+ 604us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.904126 (+ 170us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:20.905598 (+ 1472us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37484: BlockingRecv error: recv got EOF from 127.0.0.1:37484 (error 108)
14:11:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":161}
14:11:21.060 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:21.062 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:21.066 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:21.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:21.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:21.067 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:21.068218 19118 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.062136 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37496 (local address 127.18.18.126:43773)
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065416 (+ 3280us) server_negotiation.cc:206] Beginning negotiation
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065446 (+ 30us) server_negotiation.cc:399] Waiting for connection header
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065507 (+ 61us) server_negotiation.cc:407] Connection header received
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065650 (+ 143us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065680 (+ 30us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.065907 (+ 227us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:21.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.066081 (+ 174us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:21.067518 (+ 1437us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37496: BlockingRecv error: recv got EOF from 127.0.0.1:37496 (error 108)
14:11:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":176}
14:11:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:21.098469 19133 catalog_manager.cc:1261] Loaded cluster ID: a45d2b36bfa548eaad97be7e059acf10
14:11:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:21.098690 19133 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: loading cluster ID for follower catalog manager: success
14:11:21.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:21.105062 19133 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: acquiring CA information for follower catalog manager: success
14:11:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:21.108858 19133 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 11f46b1623db4bbeb64a0c50fceef758: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:11:22.719 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:22.722 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:22.725 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:22.725 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:22.725 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:22.726 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:22.726994 19148 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.721669 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46602 (local address 127.18.18.126:43773)
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.723963 (+ 2294us) server_negotiation.cc:206] Beginning negotiation
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.723989 (+ 26us) server_negotiation.cc:399] Waiting for connection header
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.724037 (+ 48us) server_negotiation.cc:407] Connection header received
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.724199 (+ 162us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.724226 (+ 27us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.724392 (+ 166us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.724590 (+ 198us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:22.726263 (+ 1673us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46602: BlockingRecv error: recv got EOF from 127.0.0.1:46602 (error 108)
14:11:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1689,"thread_start_us":1408,"threads_started":1}
14:11:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:25 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119885, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:25.214998 18858 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:25.679 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:25.681 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:25.685 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:25.685 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:25.685 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:25.686 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:25.687378 19156 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.681865 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46614 (local address 127.18.18.126:43773)
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684168 (+ 2303us) server_negotiation.cc:206] Beginning negotiation
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684204 (+ 36us) server_negotiation.cc:399] Waiting for connection header
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684276 (+ 72us) server_negotiation.cc:407] Connection header received
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684491 (+ 215us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684512 (+ 21us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684762 (+ 250us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.684941 (+ 179us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:25.686476 (+ 1535us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46614: BlockingRecv error: recv got EOF from 127.0.0.1:46614 (error 108)
14:11:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1475,"thread_start_us":1517,"threads_started":1}
14:11:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:27 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119887, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:27.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:27.209825 18950 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:27.519 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:27.521 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:27.525 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:27.525 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:27.525 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:27.526 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:27.527190 19158 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.522037 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46624 (local address 127.18.18.126:43773)
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524549 (+ 2512us) server_negotiation.cc:206] Beginning negotiation
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524578 (+ 29us) server_negotiation.cc:399] Waiting for connection header
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524628 (+ 50us) server_negotiation.cc:407] Connection header received
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524766 (+ 138us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524784 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.524949 (+ 165us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.525109 (+ 160us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:27.526454 (+ 1345us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46624: BlockingRecv error: recv got EOF from 127.0.0.1:46624 (error 108)
14:11:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1912,"thread_start_us":1597,"threads_started":1}
14:11:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:29 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119889, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:29.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:29.361879 19050 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:31.299 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:31.301 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:31.304 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:11:31.304 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc
14:11:31.305 [DEBUG - kudu-nio-0] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:11:31.305 [WARN - kudu-nio-0] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
14:11:31.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:31.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:31.306 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:31.306 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:31.307580 19168 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.301926 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46626 (local address 127.18.18.126:43773)
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303239 (+ 1313us) server_negotiation.cc:206] Beginning negotiation
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303265 (+ 26us) server_negotiation.cc:399] Waiting for connection header
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303302 (+ 37us) server_negotiation.cc:407] Connection header received
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303407 (+ 105us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303425 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303640 (+ 215us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.303846 (+ 206us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:31.306818 (+ 2972us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46626: BlockingRecv error: recv got EOF from 127.0.0.1:46626 (error 108)
14:11:31.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":900,"thread_start_us":613,"threads_started":1}
14:11:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:35 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119895, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:35.217326 18858 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:35.319 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:35.321 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:35.326 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:35.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:35.326 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:35.327 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:35.328294 19173 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.321850 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44928 (local address 127.18.18.126:43773)
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325281 (+ 3431us) server_negotiation.cc:206] Beginning negotiation
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325306 (+ 25us) server_negotiation.cc:399] Waiting for connection header
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325360 (+ 54us) server_negotiation.cc:407] Connection header received
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325524 (+ 164us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325541 (+ 17us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325744 (+ 203us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.325900 (+ 156us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:35.327369 (+ 1469us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44928: BlockingRecv error: recv got EOF from 127.0.0.1:44928 (error 108)
14:11:35.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2897,"thread_start_us":2560,"threads_started":1}
14:11:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:36.443310 17854 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7b00 after lost signal to thread 17791
14:11:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:36.443979 17854 debug-util.cc:398] Leaking SignalData structure 0x7b0800099680 after lost signal to thread 17857
14:11:37.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:37 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119897, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:37.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:37.212409 18950 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:38.840 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:38.842 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:38.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:38.846 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:38.846 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:38.848 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:38.849203 19179 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.841882 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44932 (local address 127.18.18.126:43773)
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845128 (+ 3246us) server_negotiation.cc:206] Beginning negotiation
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845147 (+ 19us) server_negotiation.cc:399] Waiting for connection header
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845184 (+ 37us) server_negotiation.cc:407] Connection header received
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845303 (+ 119us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845329 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.845587 (+ 258us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.847100 (+ 1513us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:38.847555 (+ 455us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44932: BlockingRecv error: recv got EOF from 127.0.0.1:44932 (error 108)
14:11:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":2840,"thread_start_us":2551,"threads_started":1}
14:11:39.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:39 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119899, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:39.364491 19050 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:39.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:39.767181 19002 debug-util.cc:398] Leaking SignalData structure 0x7b0800092ac0 after lost signal to thread 18934
14:11:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:39.767864 19002 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad4a0 after lost signal to thread 19005
14:11:39.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:39.800458 18349 debug-util.cc:398] Leaking SignalData structure 0x7b08000acb20 after lost signal to thread 18223
14:11:39.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:39.801594 18349 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2da0 after lost signal to thread 18352
14:11:41.460 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:41.464 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:41.474 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:11:41.475 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGY_x37/test-tmp/mini-kudu-cluster7747094218670937753/krb5kdc/krb5cc
14:11:41.476 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:11:41.476 [WARN - kudu-nio-1] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
14:11:41.476 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:41.477 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:41.477 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:41.477 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:41.479892 19184 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.464360 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:44944 (local address 127.18.18.126:43773)
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473324 (+ 8964us) server_negotiation.cc:206] Beginning negotiation
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473352 (+ 28us) server_negotiation.cc:399] Waiting for connection header
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473409 (+ 57us) server_negotiation.cc:407] Connection header received
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473560 (+ 151us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473591 (+ 31us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.473871 (+ 280us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.474078 (+ 207us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:41.478542 (+ 4464us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44944: BlockingRecv error: recv got EOF from 127.0.0.1:44944 (error 108)
14:11:41.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":8264,"thread_start_us":7948,"threads_started":1}
14:11:44.319 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:44.321 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:44.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:44.324 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:44.324 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:44.325 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:44.326390 19188 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.321385 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47008 (local address 127.18.18.126:43773)
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323420 (+ 2035us) server_negotiation.cc:206] Beginning negotiation
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323445 (+ 25us) server_negotiation.cc:399] Waiting for connection header
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323505 (+ 60us) server_negotiation.cc:407] Connection header received
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323750 (+ 245us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323776 (+ 26us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.323999 (+ 223us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.324155 (+ 156us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:44.325417 (+ 1262us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47008: BlockingRecv error: recv got EOF from 127.0.0.1:47008 (error 108)
14:11:44.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1358,"thread_start_us":1463,"threads_started":1}
14:11:45.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:45 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119905, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:45.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:45.219745 18858 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 28 14:11:47 dist-test-slave-n2gf krb5kdc[18516](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1751119907, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.18.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:11:47.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:47.214704 18950 init.cc:295] Successfully reacquired a new kerberos TGT
14:11:47.339 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:47.341 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:47.345 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:47.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:47.345 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:47.346 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:47.346962 19191 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.341833 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47010 (local address 127.18.18.126:43773)
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344206 (+ 2373us) server_negotiation.cc:206] Beginning negotiation
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344235 (+ 29us) server_negotiation.cc:399] Waiting for connection header
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344286 (+ 51us) server_negotiation.cc:407] Connection header received
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344428 (+ 142us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344446 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344621 (+ 175us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.344779 (+ 158us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:47.346169 (+ 1390us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47010: BlockingRecv error: recv got EOF from 127.0.0.1:47010 (error 108)
14:11:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1762,"thread_start_us":1492,"threads_started":1}
14:11:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:47.608424 18490 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9f00 after lost signal to thread 18364
14:11:47.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:47.609282 18490 debug-util.cc:398] Leaking SignalData structure 0x7b08000acd00 after lost signal to thread 18493
14:11:48.139 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:48.145 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:48.148 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:48.148 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:48.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:48.149 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:48.149799 19193 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.145040 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47026 (local address 127.18.18.126:43773)
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.146996 (+ 1956us) server_negotiation.cc:206] Beginning negotiation
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147015 (+ 19us) server_negotiation.cc:399] Waiting for connection header
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147050 (+ 35us) server_negotiation.cc:407] Connection header received
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147152 (+ 102us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147170 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147333 (+ 163us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.147471 (+ 138us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.149062 (+ 1591us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47026: BlockingRecv error: recv got EOF from 127.0.0.1:47026 (error 108)
14:11:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1565,"thread_start_us":1305,"threads_started":1}
14:11:48.819 [DEBUG - pool-48-thread-1] (Connection.java:748) [peer master-127.18.18.126:43773(127.18.18.126:43773)] connecting to peer
14:11:48.825 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.18.18.126:43773(127.18.18.126:43773)] Successfully connected to peer
14:11:48.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.126:43773(127.18.18.126:43773)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:48.827 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.18.126:43773: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
14:11:48.827 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807; will retry
14:11:48.829 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29049), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [357ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [375ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [399ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [400ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [401ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [403ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [406ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [462ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [463ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [481ms] refreshing cache from master, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [496ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [497ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [561ms] refreshing cache from master, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [568ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [575ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [579ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [592ms] refreshing cache from master, [592ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [609ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [641ms] refreshing cache from master, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [651ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [654ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [729ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [733ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [781ms] refreshing cache from master, [782ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [788ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [792ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [941ms] refreshing cache from master, [941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [943ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [949ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [949ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1121ms] refreshing cache from master, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1127ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1281ms] refreshing cache from master, [1281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1286ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1288ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2940ms] refreshing cache from master, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [2942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [2947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2947ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5900ms] refreshing cache from master, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [5902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [5906ms] too many traces: truncated at 100 traces, [5906ms] too many traces: truncated at 100 traces, deferred=Deferred@857018940(state=PENDING, result=null, callback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671), errback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671)))
14:11:48.833 [DEBUG - main] (Connection.java:575) [peer master-127.18.18.124:36807(127.18.18.124:36807)] disconnecting while in state READY
14:11:48.833 [DEBUG - main] (Connection.java:575) [peer master-127.18.18.126:43773(127.18.18.126:43773)] disconnecting while in state TERMINATED
14:11:48.833 [DEBUG - main] (Connection.java:575) [peer master-127.18.18.125:33259(127.18.18.125:33259)] disconnecting while in state READY
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250628 14:11:48.833561 19195 negotiation.cc:337] Failed RPC negotiation. Trace:
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.822172 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47038 (local address 127.18.18.126:43773)
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.825852 (+ 3680us) server_negotiation.cc:206] Beginning negotiation
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.825888 (+ 36us) server_negotiation.cc:399] Waiting for connection header
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.825946 (+ 58us) server_negotiation.cc:407] Connection header received
14:11:48.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.826070 (+ 124us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:11:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.826088 (+ 18us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:11:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.826278 (+ 190us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:11:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.826420 (+ 142us) server_negotiation.cc:226] Negotiated authn=SASL
14:11:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0628 14:11:48.832343 (+ 5923us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47038: BlockingRecv error: recv got EOF from 127.0.0.1:47038 (error 108)
14:11:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3130,"thread_start_us":2844,"threads_started":1}
14:11:48.835 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.18.124:36807(127.18.18.124:36807)] handling channelInactive
14:11:48.835 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.18.124:36807(127.18.18.124:36807)] cleaning up while in state READY due to: connection closed
14:11:48.837 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:48.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:48.838917 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18845
14:11:48.840 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.126:43773(127.18.18.126:43773)] handling channelInactive
14:11:48.841 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.18.18.125:33259(127.18.18.125:33259)] handling channelInactive
14:11:48.841 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.18.125:33259(127.18.18.125:33259)] cleaning up while in state READY due to: connection closed
14:11:48.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:48.866043 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18917
14:11:48.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:48.894601 18505 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 19008
14:11:50.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:11:50Z chronyd exiting
14:11:51.113 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:11:51.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.115090 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18061
14:11:51.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.147469 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18222
14:11:51.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.176299 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 18363
14:11:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.204303 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 17790
14:11:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.237493 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 17860
14:11:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250628 14:11:51.269887 17778 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskGY_x37/build/tsan/bin/kudu with pid 17935
14:11:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-28T14:11:52Z chronyd exiting
14:11:53.446 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29049), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [357ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [375ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [399ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [400ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [401ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [403ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [406ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [462ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [463ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [481ms] refreshing cache from master, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [496ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [497ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [561ms] refreshing cache from master, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [568ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [575ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [579ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [592ms] refreshing cache from master, [592ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [609ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [641ms] refreshing cache from master, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [651ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [654ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [729ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [733ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [781ms] refreshing cache from master, [782ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [788ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [792ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [941ms] refreshing cache from master, [941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [943ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [949ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [949ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1121ms] refreshing cache from master, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1127ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1281ms] refreshing cache from master, [1281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1286ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1288ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2940ms] refreshing cache from master, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [2942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [2947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2947ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5900ms] refreshing cache from master, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [5902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [5906ms] too many traces: truncated at 100 traces, [5906ms] too many traces: truncated at 100 traces, deferred=Deferred@857018940(state=PENDING, result=null, callback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671), errback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29049), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [357ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [375ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [399ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [400ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [401ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [403ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [406ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [462ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [463ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [481ms] refreshing cache from master, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [496ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [497ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [561ms] refreshing cache from master, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [568ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [575ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [579ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [592ms] refreshing cache from master, [592ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [609ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [641ms] refreshing cache from master, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [651ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [654ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [729ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [733ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [781ms] refreshing cache from master, [782ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [788ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [792ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [941ms] refreshing cache from master, [941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [943ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [949ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [949ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1121ms] refreshing cache from master, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1127ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1281ms] refreshing cache from master, [1281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1286ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1288ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2940ms] refreshing cache from master, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [2942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [2947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2947ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5900ms] refreshing cache from master, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [5902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [5906ms] too many traces: truncated at 100 traces, [5906ms] too many traces: truncated at 100 traces, deferred=Deferred@857018940(state=PENDING, result=null, callback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671), errback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more
14:11:53.648 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: giving up after 1 attempts
14:11:53.648 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29049), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [357ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [375ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [399ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [400ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [401ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [403ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [406ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [462ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [463ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [481ms] refreshing cache from master, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [496ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [497ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [561ms] refreshing cache from master, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [568ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [575ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [579ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [592ms] refreshing cache from master, [592ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [609ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [641ms] refreshing cache from master, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [651ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [654ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [729ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [733ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [781ms] refreshing cache from master, [782ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [788ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [792ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [941ms] refreshing cache from master, [941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [943ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [949ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [949ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1121ms] refreshing cache from master, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1127ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1281ms] refreshing cache from master, [1281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1286ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1288ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2940ms] refreshing cache from master, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [2942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [2947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2947ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5900ms] refreshing cache from master, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [5902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [5906ms] too many traces: truncated at 100 traces, [5906ms] too many traces: truncated at 100 traces, deferred=Deferred@857018940(state=PENDING, result=null, callback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671), errback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=29049), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [14ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [357ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [375ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [399ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [400ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [401ms] refreshing cache from master, [401ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [402ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [403ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [406ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [438ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [462ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [463ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [481ms] refreshing cache from master, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [483ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [496ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [497ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [546ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [546ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [561ms] refreshing cache from master, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [563ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [568ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [575ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [579ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [580ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [592ms] refreshing cache from master, [592ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [593ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [599ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [609ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [641ms] refreshing cache from master, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [642ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [651ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [652ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [654ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [654ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [721ms] refreshing cache from master, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [722ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [729ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [733ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [781ms] refreshing cache from master, [782ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [788ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [789ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [792ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [792ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [941ms] refreshing cache from master, [941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [943ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [949ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [949ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1121ms] refreshing cache from master, [1121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1126ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1127ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1281ms] refreshing cache from master, [1281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [1282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [1283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [1286ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [1288ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2940ms] refreshing cache from master, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [2941ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [2942ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.125:33259: OK, [2946ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.124:36807: OK, [2947ms] Sub RPC ConnectToMaster: received response from server master-127.18.18.126:43773: Not authorized: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [2947ms] delaying RPC due to: Service unavailable: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available, [5900ms] refreshing cache from master, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.126:43773, [5901ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.125:33259, [5902ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.18.124:36807, [5906ms] too many traces: truncated at 100 traces, [5906ms] too many traces: truncated at 100 traces, deferred=Deferred@857018940(state=PENDING, result=null, callback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671), errback=(continuation of Deferred@1677752658 after retry RPC after error@1694865605) -> (continuation of Deferred@336304963 after retry RPC after error@261422860) -> (continuation of Deferred@1757007173 after retry RPC after error@557036327) -> (continuation of Deferred@1764419009 after retry RPC after error@1097730935) -> (continuation of Deferred@1729832442 after retry RPC after error@1527580853) -> (continuation of Deferred@1706556152 after retry RPC after error@1412657290) -> (continuation of Deferred@1790552835 after retry RPC after error@1497517633) -> (continuation of Deferred@66588587 after retry RPC after error@1275107454) -> (continuation of Deferred@1799799630 after retry RPC after error@549300257) -> (continuation of Deferred@1677426378 after retry RPC after error@728811939) -> (continuation of Deferred@1995518768 after retry RPC after error@1002463599) -> (continuation of Deferred@66409971 after retry RPC after error@786803199) -> (continuation of Deferred@1366452128 after retry RPC after error@1761722360) -> (continuation of Deferred@1081226726 after retry RPC after error@1130092102) -> (continuation of Deferred@30865130 after retry RPC after error@1707894615) -> (continuation of Deferred@876655373 after retry RPC after error@691321381) -> (continuation of Deferred@526608463 after retry RPC after error@429085227) -> (continuation of Deferred@679101135 after retry RPC after error@613621879) -> (continuation of Deferred@540411322 after retry RPC after error@590809501) -> (continuation of Deferred@1495358222 after retry RPC after error@1686919365) -> (continuation of Deferred@1946655291 after retry RPC after error@1877417671)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testDoNotSwitchPrincipalsInExistingClient(TestSecurity.java:366)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.18.126:43773,127.18.18.125:33259,127.18.18.124:36807) has no leader. Exceptions received: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because no token is available
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
... 30 more