Pull OPUS-100 en-zh for SFT instead of a checked-in jsonl
train_sft --data opus-100 streams Helsinki-NLP/opus-100, writes both directions, and skips frozen eval sentences. Runtime cache stays under data/sft/ (gitignored).
This commit is contained in:
@@ -1,4 +1,11 @@
|
||||
from kda.training.data import IGNORE_INDEX, collate_sft, encode_sft_row, load_sft_rows
|
||||
from kda.training.data import (
|
||||
IGNORE_INDEX,
|
||||
collate_sft,
|
||||
encode_sft_row,
|
||||
fetch_opus100_enzh,
|
||||
load_sft_rows,
|
||||
resolve_sft_rows,
|
||||
)
|
||||
from kda.training.prompts import instruction_prompt
|
||||
|
||||
|
||||
@@ -42,6 +49,36 @@ def test_collate_and_jsonl(tmp_path):
|
||||
assert (y == IGNORE_INDEX).any()
|
||||
|
||||
|
||||
def test_resolve_sft_rows_reads_local_jsonl(tmp_path):
|
||||
path = tmp_path / "bitext.jsonl"
|
||||
path.write_text(
|
||||
'{"src": "你好", "tgt": "Hello", "target_lang": "en"}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
rows = resolve_sft_rows(str(path))
|
||||
assert rows == [{"src": "你好", "tgt": "Hello", "target_lang": "en"}]
|
||||
|
||||
|
||||
def test_fetch_opus_skips_eval_sentences(tmp_path, monkeypatch):
|
||||
eval_dir = tmp_path / "eval"
|
||||
eval_dir.mkdir()
|
||||
(eval_dir / "zh2en.src.txt").write_text("禁止句\n", encoding="utf-8")
|
||||
|
||||
class _DS:
|
||||
def __iter__(self):
|
||||
yield {"translation": {"en": "Hello", "zh": "你好"}}
|
||||
yield {"translation": {"en": "skip", "zh": "禁止句"}}
|
||||
yield {"translation": {"en": "Thanks", "zh": "谢谢"}}
|
||||
|
||||
fake = type("datasets", (), {"load_dataset": staticmethod(lambda *a, **k: _DS())})
|
||||
monkeypatch.setitem(__import__("sys").modules, "datasets", fake)
|
||||
rows = fetch_opus100_enzh(10, cache_dir=tmp_path / "sft", eval_dir=eval_dir)
|
||||
srcs = {r["src"] for r in rows}
|
||||
assert "禁止句" not in srcs
|
||||
assert "你好" in srcs and "Hello" in srcs
|
||||
assert "谢谢" in srcs and "Thanks" in srcs
|
||||
|
||||
|
||||
def test_toy_sft_file_parses():
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
Reference in New Issue
Block a user