Fix chrF fallback so English wiki garbage fails translation_success
Sacrebleu errors used to fall back to set-overlap unigrams, so any English hyp scored ~70–80 against English refs and SFT reported success 1.0. Use count-based char n-grams, keep BLEU failures from clobbering chrF, and print a few hyps during eval.
This commit is contained in:
@@ -67,14 +67,18 @@ def evaluate_pairs(
|
||||
want = "zh" if target_lang.startswith("zh") else "en"
|
||||
lang_ok += int(_detect_lang(hyp) == want)
|
||||
chrf_sum += _chrf(hyp, ref)
|
||||
corpus = {}
|
||||
corpus = {"chrf": chrf_sum / max(n, 1), "bleu": None}
|
||||
try:
|
||||
from sacrebleu.metrics import BLEU, CHRF
|
||||
from sacrebleu.metrics import CHRF
|
||||
|
||||
corpus["chrf"] = float(CHRF(word_order=2).corpus_score(hyps, [refs[:n]]).score)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from sacrebleu.metrics import BLEU
|
||||
|
||||
corpus["bleu"] = float(BLEU().corpus_score(hyps, [refs[:n]]).score)
|
||||
except Exception:
|
||||
corpus["chrf"] = chrf_sum / max(n, 1)
|
||||
corpus["bleu"] = None
|
||||
return {
|
||||
"n": n,
|
||||
@@ -131,6 +135,8 @@ def main() -> None:
|
||||
)
|
||||
printable = {k: v for k, v in out.items() if k != "hyps"}
|
||||
print(json.dumps(printable, ensure_ascii=False, indent=2))
|
||||
for i, hyp in enumerate(out["hyps"][: min(5, out["n"])]):
|
||||
print(f" [{i}] {hyp}")
|
||||
elif not args.prefix:
|
||||
raise SystemExit("pass --prefix and/or --src + --ref")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user