KV Cache: The Heart of LLM Serving

# ——————————————————————————————————————
# SYMPTOM: P99 TTFT spiking at high concurrency; users see latency
# GOAL: Identify whether bottleneck is KV fragmentation, memory pressure,
#       or prefix cache misses
# ——————————————————————————————————————

# STEP 1: Check HBM utilization and KV cache occupancy
nvidia-smi --query-gpu=memory.used,memory.free,utilization.gpu \
    --format=csv,noheader -l 1
# If memory.used consistently > 95% of 80 GB: KV pressure
# If GPU util drops to <30% during decode: not KV-bound, likely overhead

# STEP 2: vLLM built-in KV cache metrics (Prometheus endpoint)
curl -s http://localhost:8000/metrics | grep -E "kv_cache|prefix"
# vllm:gpu_cache_usage_perc      — fraction of KV blocks in use (target <90%)
# vllm:prefix_cache_hit_rate     — prefix cache efficiency (target >40%)
# vllm:num_preemptions_total     — number of requests preempted (KV evictions)
# If preemptions > 0: KV pressure; reduce batch size or enable FP8 KV
# If prefix hit rate <20%: disable prefix cache (overhead > benefit)

# STEP 3: Compute theoretical KV capacity vs actual workload
python3 -c "
L, H_kv, d_head = 80, 8, 128  # LLaMA-3 70B
dtype_bytes = 1  # FP8
kv_per_token_mb = 2 * L * H_kv * d_head * dtype_bytes / 1e6

weights_gb = 35  # FP8 weights, TP=2 per GPU
headroom_gb = 80 - weights_gb - 5  # 5 GB runtime overhead

for ctx in [1024, 4096, 8192, 32768]:
    per_req_gb = kv_per_token_mb * ctx / 1024
    max_users = headroom_gb * 1024 / (kv_per_token_mb * ctx)
    print(f'{ctx:6d} tokens: {per_req_gb:.2f} GB/req, {max_users:.0f} max users')
"
#   1024 tokens: 0.16 GB/req, 250 max users
#   4096 tokens: 0.64 GB/req,  62 max users
#   8192 tokens: 1.28 GB/req,  31 max users
#  32768 tokens: 5.12 GB/req,   8 max users

# STEP 4: Diagnose prefix cache hit rate and sizing
python3 -c "
# Size the prefix cache for a chatbot system prompt workload
S = 1024   # shared system prompt tokens
U = 128    # average unique user tokens per request
h = 0.80   # target hit rate
kv_per_token_mb = 0.160  # FP8 KV, LLaMA-3 70B

reduction = 1 - (S*(1-h) + U) / (S + U)
print(f'Prefill reduction at h={h}: {reduction:.1%}')
print(f'TTFT speedup: {1/(1-reduction):.1f}x')

# Minimum cache size for N concurrent system prompts
N_prompts = 50  # distinct system prompts in deployment
min_cache_mb = N_prompts * S * kv_per_token_mb
print(f'Minimum prefix cache: {min_cache_mb:.0f} MB ({min_cache_mb/1024:.1f} GB)')
print(f'Recommended (2x buffer): {2*min_cache_mb/1024:.1f} GB')
"
# Prefill reduction at h=0.80: 71.1%
# TTFT speedup: 3.5x
# Minimum prefix cache: 8192 MB (8.0 GB)
# Recommended (2x buffer): 16.0 GB

# STEP 5: Check KV quantization quality before enabling
lm_eval --model hf \
  --model_args pretrained=meta-llama/Meta-Llama-3-70B \
  --tasks mmlu,humaneval,gsm8k \
  --num_fewshot 5 \
  --output_path results_baseline.json

# Then serve with FP8 KV and re-evaluate:
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3-70B \
  --kv-cache-dtype fp8 \
  --quantization fp8 &

# Compare: if any benchmark drops >1.5% → consider INT8 instead of FP8 KV
# HumanEval drop >2% → INT4 KV is contraindicated for this model
# Expected FP8 KV quality change: all benchmarks within 0.2%
# If seeing larger drops: check per-head vs per-tensor scaling config