Fit 0.5b training on 32GB: SDPA MLA, block checkpoint, chunked CE

Whole-mixer checkpoint plus T×T MLA scores OOM'd a 31GB GPU on backward.
Checkpoint each AttnRes block, run absorbed MLA through SDPA, and compute
CE in vocab chunks so [B,T,V] logits are never materialized.

--max-tokens is now the training budget; default --steps 2000 no longer
caps a 1B-token run at 250 optimizer steps.
This commit is contained in:
dela
2026-08-25 20:09:27 +08:00
parent 7a12f61de1
commit 49aede9cb2
7 changed files with 117 additions and 36 deletions
+11 -3
View File
@@ -9,13 +9,21 @@ def test_warmup_then_cosine_floor():
assert abs(end - 0.1) < 1e-6
def test_horizon_prefers_the_earlier_stop():
# 8.2M tokens @ batch 2 seq 2048 acc 8 -> 250 opt
def test_horizon_max_tokens_overrides_micro_cap():
# 8.2M tokens @ batch 2 seq 2048 acc 8 -> 250 opt even if --steps is larger
opt_from_tokens = total_opt_steps(
max_tokens=8_192_000, max_micro=10_000, batch=2, seq_len=2048, grad_acc=8
)
assert opt_from_tokens == 250
# 1B-token run must not inherit the default --steps 2000 cap (250 opt)
opt_1b = total_opt_steps(
max_tokens=10**9, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
)
assert opt_1b == 30518
def test_horizon_micro_when_tokens_unset():
opt_from_micro = total_opt_steps(
max_tokens=10**12, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
max_tokens=None, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
)
assert opt_from_micro == 250