Fix chrF fallback so English wiki garbage fails translation_success

Sacrebleu errors used to fall back to set-overlap unigrams, so any
English hyp scored ~70–80 against English refs and SFT reported
success 1.0. Use count-based char n-grams, keep BLEU failures from
clobbering chrF, and print a few hyps during eval.
This commit is contained in:
dela
2026-08-26 14:23:22 +08:00
parent 5a7d949b01
commit 5cc0555563
4 changed files with 48 additions and 14 deletions
+9 -3
View File
@@ -67,14 +67,18 @@ def evaluate_pairs(
want = "zh" if target_lang.startswith("zh") else "en"
lang_ok += int(_detect_lang(hyp) == want)
chrf_sum += _chrf(hyp, ref)
corpus = {}
corpus = {"chrf": chrf_sum / max(n, 1), "bleu": None}
try:
from sacrebleu.metrics import BLEU, CHRF
from sacrebleu.metrics import CHRF
corpus["chrf"] = float(CHRF(word_order=2).corpus_score(hyps, [refs[:n]]).score)
except Exception:
pass
try:
from sacrebleu.metrics import BLEU
corpus["bleu"] = float(BLEU().corpus_score(hyps, [refs[:n]]).score)
except Exception:
corpus["chrf"] = chrf_sum / max(n, 1)
corpus["bleu"] = None
return {
"n": n,
@@ -131,6 +135,8 @@ def main() -> None:
)
printable = {k: v for k, v in out.items() if k != "hyps"}
print(json.dumps(printable, ensure_ascii=False, indent=2))
for i, hyp in enumerate(out["hyps"][: min(5, out["n"])]):
print(f" [{i}] {hyp}")
elif not args.prefix:
raise SystemExit("pass --prefix and/or --src + --ref")