Training Systems at Scale

# ————————————————————————————————————————
# SYMPTOM: MFU = 28%; GPU SM utilization shows 95% in Nsight Compute
# (These two numbers are NOT contradictory — see Step 2 for why)
# ————————————————————————————————————————

# STEP 1: Compute true MFU from training log
python3 -c "
tokens_per_sec = 42500   # from training log (global tokens/sec)
P = 30e9                 # model parameters
N = 256                  # GPUs
F_peak = 989e12          # H100 BF16 TFLOPS
MFU = (tokens_per_sec * 6 * P) / (N * F_peak)
print(f'MFU = {MFU:.1%}')
"
# MFU = 28.7%  -- confirmed low. Now find which layer is the bottleneck.

# STEP 2: Why 95% SM util != 95% MFU?
# SM "active" = executing SOMETHING. Could be FP32 CUDA cores (33 TFLOPS)
# not BF16 Tensor Cores (989 TFLOPS). Check TC utilization specifically:
ncu --section ComputeWorkloadAnalysis \
    --kernel-name ".*gemm.*" --launch-skip 10 --launch-count 2 \
    python train.py --steps 3
# sm__pipe_tensor_cycles_active.avg.pct_of_peak: 31%  <— LOW!
# Cause: weight dimensions not aligned to Tensor Core requirements (mult of 16)
# Fix: ensure all hidden/FFN dims are multiples of 64

# STEP 3: Check gradient communication overlap with nsys
nsys profile \
    --trace=cuda,nvtx,nccl \
    --output=/tmp/train_trace \
    --duration=30 \
    python train.py --steps 10
# Open in Nsight Systems GUI. Look for:
# GOOD: NCCL AllReduce timeline rows overlap with CUDA kernel rows
# BAD:  NCCL rows appear entirely AFTER compute rows (sequential!)
# BAD signature: FSDP AllGather completes, then backward starts (not pipelined)

# STEP 4: Verify FSDP overlap is configured correctly
python3 -c "
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
from torch.distributed.fsdp import BackwardPrefetch
# Check your model config has these set:
print('Required FSDP overlap config:')
print('  backward_prefetch=BackwardPrefetch.BACKWARD_PRE')
print('  forward_prefetch=True')
print('  limit_all_gathers=True  # memory safety')
"

# STEP 5: Check pipeline bubble fraction
python3 -c "
PP = 4    # your pipeline stages
M  = 8    # micro-batches per step (grad_accum * DP / PP)
bubble = (PP-1) / (PP-1 + M)
print(f'Bubble fraction: {bubble:.1%}')
target_M = (PP-1) * 10 - (PP-1) + 1  # for <10% bubble: M >= 4*PP
print(f'Target M for <10% bubble: {4*PP}')
print(f'Action: set gradient_accumulation_steps = {4*PP * PP // max(1,16)}')
"
# Bubble fraction: 27.3%  — VERY HIGH
# Target M for <10% bubble: 16
# Action: increase gradient_accumulation_steps

# STEP 6: Check data pipeline is not starving the GPU
watch -n 0.5 "nvidia-smi --query-gpu=utilization.gpu,utilization.memory \
    --format=csv,noheader"
# If GPU util drops to 0% periodically between steps: DataLoader starvation
# Fix: increase num_workers, use larger shard files, enable pin_memory=True

# STEP 7: Diagnose silent data corruption (run periodically)
python3 -c "
# In your training loop, add gradient norm monitoring:
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
if torch.isnan(grad_norm) or grad_norm > 5.0:
    print(f'WARNING: gradient spike at step {step}: {grad_norm:.2f}')
    # Skip optimizer step; increment spike counter
    # If >3 spikes in 100 steps: trigger checkpoint rollback
"
# ECC error check: nvidia-smi --query-gpu=ecc.errors.uncorrected.volatile.total
# If XID 79 (DBE): immediately drain that GPU; checkpoint current state