"""Oswald's scoring, the same as the V3 evaluation (scripts/qwen3_asr/evaluate.py and parakeet_v3/evaluate_v3.py). Every text, reference and hypothesis, is read back to its spoken form (itn.spoken: "1500 goud" -> "mil senksan goud") and stripped of case, punctuation and accents (normalize.metric_form). So V1, which writes numbers in words, and V2/V3, which write digits, are compared on the same words, and รจ/e never decides an error. """ import difflib import html import re import jiwer from itn import itn, spoken from normalize import metric_form, normalize def mform(text): return metric_form(spoken(text or "")) def target(text): """The dataset's sentence as V2/V3 were taught to write it: lowercase, no punctuation, numbers as digits.""" return itn(normalize(text or "")) def rates(refs, hyps): pairs = [(mform(r), mform(h)) for r, h in zip(refs, hyps)] pairs = [(r, h) for r, h in pairs if r] if not pairs: return None, None return jiwer.wer([p[0] for p in pairs], [p[1] for p in pairs]), jiwer.cer([p[0] for p in pairs], [p[1] for p in pairs]) def clip_wer(ref, hyp): r = mform(ref) return jiwer.wer(r, mform(hyp)) if r else None # Number words in metric form. A tens word carries its unit glued on (vennsenk, katrevendis). _UNITS = {"zewo", "en", "de", "twa", "kat", "senk", "sis", "set", "uit", "nef", "dis", "onz", "douz", "trez", "katoz", "kenz", "sez", "diset", "dizuit", "diznef", "san", "mil", "milyon", "milya", "premye", "desan", "twasan", "katsan", "senksan", "sisan", "setsan", "uisan", "nefsan", "pwen", "vigil"} _TENS = re.compile(r"^(ven|venn|vent|trant|trann|karant|karann|senkant|senkann|swasant|swasann|katreven)\w*$") # These are also ordinary words (of, card, without, ...): alone they are not counted as a number. _AMBIGUOUS = {"en", "de", "twa", "kat", "san", "sis", "set", "premye", "pwen", "vigil"} def _is_num(w): return w in _UNITS or bool(_TENS.match(w)) or (w.endswith("yem") and len(w) > 4) def number_spans(text): words, spans, cur = mform(text).split(), [], [] for w in words + [""]: if w and _is_num(w): cur.append(w) continue if cur and (len(cur) > 1 or cur[0] not in _AMBIGUOUS): spans.append(" ".join(cur)) cur = [] return spans def number_accuracy(refs, hyps): """Share of the reference's numbers found whole in the transcript ("de mil vennkat" as "de mil kat" is wrong).""" hit = total = 0 for r, h in zip(refs, hyps): hyp = " " + " ".join(mform(h).split()) + " " for span in number_spans(r): total += 1 hit += f" {span} " in hyp return (hit / total if total else None), total def diff_html(ref, hyp): """The hypothesis as scored, word by word: missing words struck through in red, extra or wrong ones in orange.""" r, h = mform(ref).split(), mform(hyp).split() out = [] for op, a, b, c, d in difflib.SequenceMatcher(a=r, b=h, autojunk=False).get_opcodes(): if op == "equal": out.append(html.escape(" ".join(h[c:d]))) continue if b > a: out.append(f'{html.escape(" ".join(r[a:b]))}') if d > c: out.append(f'{html.escape(" ".join(h[c:d]))}') return " ".join(out)