Fix chrF fallback so English wiki garbage fails translation_success
Sacrebleu errors used to fall back to set-overlap unigrams, so any English hyp scored ~70–80 against English refs and SFT reported success 1.0. Use count-based char n-grams, keep BLEU failures from clobbering chrF, and print a few hyps during eval.
This commit is contained in:
+27
-11
@@ -28,8 +28,33 @@ def _detect_lang(text: str) -> str | None:
|
||||
return tag[:2]
|
||||
|
||||
|
||||
def _chrf_ngram(hyp: str, ref: str, max_n: int = 4) -> float:
|
||||
"""Count-based char n-gram F (β=2), 0–100. Not set-overlap unigrams."""
|
||||
from collections import Counter
|
||||
|
||||
hyp, ref = hyp.strip(), ref.strip()
|
||||
if not hyp or not ref:
|
||||
return 0.0
|
||||
scores: list[float] = []
|
||||
for n in range(1, max_n + 1):
|
||||
if len(hyp) < n or len(ref) < n:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
hc = Counter(hyp[i : i + n] for i in range(len(hyp) - n + 1))
|
||||
rc = Counter(ref[i : i + n] for i in range(len(ref) - n + 1))
|
||||
overlap = sum((hc & rc).values())
|
||||
prec = overlap / max(sum(hc.values()), 1)
|
||||
rec = overlap / max(sum(rc.values()), 1)
|
||||
if prec + rec == 0:
|
||||
scores.append(0.0)
|
||||
continue
|
||||
beta2 = 4.0
|
||||
scores.append((1.0 + beta2) * prec * rec / (beta2 * prec + rec))
|
||||
return 100.0 * sum(scores) / max(len(scores), 1)
|
||||
|
||||
|
||||
def _chrf(hyp: str, ref: str) -> float:
|
||||
"""chrF++ in 0–100. Falls back to char unigram F if sacrebleu is missing."""
|
||||
"""chrF++ in 0–100. Falls back to count-based char n-grams if sacrebleu is missing."""
|
||||
if not hyp or not ref:
|
||||
return 0.0
|
||||
try:
|
||||
@@ -37,16 +62,7 @@ def _chrf(hyp: str, ref: str) -> float:
|
||||
|
||||
return float(CHRF(word_order=2).sentence_score(hyp, [ref]).score)
|
||||
except Exception:
|
||||
hyp_c, ref_c = list(hyp), list(ref)
|
||||
if not hyp_c:
|
||||
return 0.0
|
||||
ref_set = set(ref_c)
|
||||
overlap = sum(1 for c in hyp_c if c in ref_set)
|
||||
prec = overlap / len(hyp_c)
|
||||
rec = overlap / max(len(ref_c), 1)
|
||||
if prec + rec == 0:
|
||||
return 0.0
|
||||
return 100.0 * 2 * prec * rec / (prec + rec)
|
||||
return _chrf_ngram(hyp, ref)
|
||||
|
||||
|
||||
def translation_success(
|
||||
|
||||
Reference in New Issue
Block a user