Fit 0.5b training on 32GB: SDPA MLA, block checkpoint, chunked CE
Whole-mixer checkpoint plus T×T MLA scores OOM'd a 31GB GPU on backward. Checkpoint each AttnRes block, run absorbed MLA through SDPA, and compute CE in vocab chunks so [B,T,V] logits are never materialized. --max-tokens is now the training budget; default --steps 2000 no longer caps a 1B-token run at 250 optimizer steps.
This commit is contained in:
@@ -9,13 +9,21 @@ def test_warmup_then_cosine_floor():
|
||||
assert abs(end - 0.1) < 1e-6
|
||||
|
||||
|
||||
def test_horizon_prefers_the_earlier_stop():
|
||||
# 8.2M tokens @ batch 2 seq 2048 acc 8 -> 250 opt
|
||||
def test_horizon_max_tokens_overrides_micro_cap():
|
||||
# 8.2M tokens @ batch 2 seq 2048 acc 8 -> 250 opt even if --steps is larger
|
||||
opt_from_tokens = total_opt_steps(
|
||||
max_tokens=8_192_000, max_micro=10_000, batch=2, seq_len=2048, grad_acc=8
|
||||
)
|
||||
assert opt_from_tokens == 250
|
||||
# 1B-token run must not inherit the default --steps 2000 cap (250 opt)
|
||||
opt_1b = total_opt_steps(
|
||||
max_tokens=10**9, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
|
||||
)
|
||||
assert opt_1b == 30518
|
||||
|
||||
|
||||
def test_horizon_micro_when_tokens_unset():
|
||||
opt_from_micro = total_opt_steps(
|
||||
max_tokens=10**12, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
|
||||
max_tokens=None, max_micro=2000, batch=2, seq_len=2048, grad_acc=8
|
||||
)
|
||||
assert opt_from_micro == 250
|
||||
|
||||
Reference in New Issue
Block a user