Pull OPUS-100 en-zh for SFT instead of a checked-in jsonl

train_sft --data opus-100 streams Helsinki-NLP/opus-100, writes both
directions, and skips frozen eval sentences. Runtime cache stays under
data/sft/ (gitignored).
This commit is contained in:
dela
2026-08-25 21:39:58 +08:00
parent 53d0f4b17a
commit e7185cbf49
3 changed files with 172 additions and 8 deletions
+38 -1
View File
@@ -1,4 +1,11 @@
from kda.training.data import IGNORE_INDEX, collate_sft, encode_sft_row, load_sft_rows
from kda.training.data import (
IGNORE_INDEX,
collate_sft,
encode_sft_row,
fetch_opus100_enzh,
load_sft_rows,
resolve_sft_rows,
)
from kda.training.prompts import instruction_prompt
@@ -42,6 +49,36 @@ def test_collate_and_jsonl(tmp_path):
assert (y == IGNORE_INDEX).any()
def test_resolve_sft_rows_reads_local_jsonl(tmp_path):
path = tmp_path / "bitext.jsonl"
path.write_text(
'{"src": "你好", "tgt": "Hello", "target_lang": "en"}\n',
encoding="utf-8",
)
rows = resolve_sft_rows(str(path))
assert rows == [{"src": "你好", "tgt": "Hello", "target_lang": "en"}]
def test_fetch_opus_skips_eval_sentences(tmp_path, monkeypatch):
eval_dir = tmp_path / "eval"
eval_dir.mkdir()
(eval_dir / "zh2en.src.txt").write_text("禁止句\n", encoding="utf-8")
class _DS:
def __iter__(self):
yield {"translation": {"en": "Hello", "zh": "你好"}}
yield {"translation": {"en": "skip", "zh": "禁止句"}}
yield {"translation": {"en": "Thanks", "zh": "谢谢"}}
fake = type("datasets", (), {"load_dataset": staticmethod(lambda *a, **k: _DS())})
monkeypatch.setitem(__import__("sys").modules, "datasets", fake)
rows = fetch_opus100_enzh(10, cache_dir=tmp_path / "sft", eval_dir=eval_dir)
srcs = {r["src"] for r in rows}
assert "禁止句" not in srcs
assert "你好" in srcs and "Hello" in srcs
assert "谢谢" in srcs and "Thanks" in srcs
def test_toy_sft_file_parses():
from pathlib import Path