Initial K3 snapshot: 0.5B KDA/MLA/MoE train path

Standalone tree split from LLMRL/projects/kda. Includes Triton dt_bias
backward fix, train_k3 --preset 0.5b, SFT, Docker runtime, and tests.
This commit is contained in:
dela
2026-08-25 14:43:17 +08:00
commit 584f7e9e73
140 changed files with 21592 additions and 0 deletions
+29
View File
@@ -0,0 +1,29 @@
# GPU train / eval against the image in Dockerfile.
# docker compose run --rm train python train_k3.py --preset toy
# docker compose run --rm train swanlab ping
# docker compose run --rm train python -m kda.training.eval_mt --ckpt ...
#
# Secrets and corpora stay on the host.
services:
train:
build: .
image: kda:latest
gpus: all
ipc: host
shm_size: "2gb"
working_dir: /workspace/kda
environment:
SWANLAB_API_KEY: ${SWANLAB_API_KEY:-}
HF_HOME: /cache/huggingface
HUGGINGFACE_HUB_CACHE: /cache/huggingface
volumes:
- ./ckpts:/workspace/kda/ckpts
- ./data:/workspace/kda/data
- ${PRETRAIN_DATA:-./data/pretrain}:/data/pretrain:ro
- ${EVAL_DATA:-./data/eval}:/data/eval:ro
- ${SFT_DATA:-./data/sft}:/data/sft:ro
- hf-cache:/cache/huggingface
volumes:
hf-cache: