dela
5cc0555563
Sacrebleu errors used to fall back to set-overlap unigrams, so any English hyp scored ~70–80 against English refs and SFT reported success 1.0. Use count-based char n-grams, keep BLEU failures from clobbering chrF, and print a few hyps during eval.
55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
"""translation_success and eval helpers (no GPU, no FLORES download)."""
|
|
from kda.training.success import translation_success
|
|
|
|
|
|
def test_empty_and_copy_fail():
|
|
src = "人工智能的发展改变了世界。"
|
|
assert translation_success(src, "", ref="The development of AI changed the world.", target_lang="en") is False
|
|
assert translation_success(src, src, ref="The development of AI changed the world.", target_lang="en") is False
|
|
|
|
|
|
def test_wrong_language_fails():
|
|
src = "The cat sat on the mat."
|
|
hyp = "The cat sat on the mat and smiled."
|
|
ref = "猫坐在垫子上。"
|
|
assert translation_success(src, hyp, ref, target_lang="zh") is False
|
|
|
|
|
|
def test_instruction_leak_fails():
|
|
src = "Hello"
|
|
hyp = "翻译如下：你好"
|
|
assert translation_success(src, hyp, ref="你好", target_lang="zh") is False
|
|
|
|
|
|
def test_good_zh2en_passes():
|
|
src = "今天天气很好。"
|
|
hyp = "The weather is very nice today."
|
|
ref = "The weather is very nice today."
|
|
assert translation_success(src, hyp, ref, target_lang="en") is True
|
|
|
|
|
|
def test_english_wiki_garbage_does_not_pass_zh2en():
|
|
from kda.training.success import _chrf
|
|
|
|
src = "今天天气很好。"
|
|
hyp = "The first one's the time."
|
|
ref = "The weather is very nice today."
|
|
assert _chrf(hyp, ref) < 40.0
|
|
assert translation_success(src, hyp, ref, target_lang="en") is False
|
|
|
|
|
|
def test_container_help_exits_2():
|
|
import importlib.util
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
path = Path(__file__).resolve().parents[2] / "scripts" / "container_help.py"
|
|
spec = importlib.util.spec_from_file_location("container_help", path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
assert spec.loader is not None
|
|
spec.loader.exec_module(mod)
|
|
with pytest.raises(SystemExit) as ei:
|
|
mod.main()
|
|
assert ei.value.code == 2
|