Spaces:
Running on Zero
Running on Zero
| """Oswald's scoring, the same as the V3 evaluation (scripts/qwen3_asr/evaluate.py and parakeet_v3/evaluate_v3.py). | |
| Every text, reference and hypothesis, is read back to its spoken form (itn.spoken: "1500 goud" -> "mil senksan | |
| goud") and stripped of case, punctuation and accents (normalize.metric_form). So V1, which writes numbers in words, | |
| and V2/V3, which write digits, are compared on the same words, and è/e never decides an error. | |
| """ | |
| import difflib | |
| import html | |
| import re | |
| import jiwer | |
| from itn import itn, spoken | |
| from normalize import metric_form, normalize | |
| def mform(text): | |
| return metric_form(spoken(text or "")) | |
| def target(text): | |
| """The dataset's sentence as V2/V3 were taught to write it: lowercase, no punctuation, numbers as digits.""" | |
| return itn(normalize(text or "")) | |
| def rates(refs, hyps): | |
| pairs = [(mform(r), mform(h)) for r, h in zip(refs, hyps)] | |
| pairs = [(r, h) for r, h in pairs if r] | |
| if not pairs: | |
| return None, None | |
| return jiwer.wer([p[0] for p in pairs], [p[1] for p in pairs]), jiwer.cer([p[0] for p in pairs], [p[1] for p in pairs]) | |
| def clip_wer(ref, hyp): | |
| r = mform(ref) | |
| return jiwer.wer(r, mform(hyp)) if r else None | |
| # Number words in metric form. A tens word carries its unit glued on (vennsenk, katrevendis). | |
| _UNITS = {"zewo", "en", "de", "twa", "kat", "senk", "sis", "set", "uit", "nef", "dis", "onz", "douz", "trez", | |
| "katoz", "kenz", "sez", "diset", "dizuit", "diznef", "san", "mil", "milyon", "milya", "premye", | |
| "desan", "twasan", "katsan", "senksan", "sisan", "setsan", "uisan", "nefsan", "pwen", "vigil"} | |
| _TENS = re.compile(r"^(ven|venn|vent|trant|trann|karant|karann|senkant|senkann|swasant|swasann|katreven)\w*$") | |
| # These are also ordinary words (of, card, without, ...): alone they are not counted as a number. | |
| _AMBIGUOUS = {"en", "de", "twa", "kat", "san", "sis", "set", "premye", "pwen", "vigil"} | |
| def _is_num(w): | |
| return w in _UNITS or bool(_TENS.match(w)) or (w.endswith("yem") and len(w) > 4) | |
| def number_spans(text): | |
| words, spans, cur = mform(text).split(), [], [] | |
| for w in words + [""]: | |
| if w and _is_num(w): | |
| cur.append(w) | |
| continue | |
| if cur and (len(cur) > 1 or cur[0] not in _AMBIGUOUS): | |
| spans.append(" ".join(cur)) | |
| cur = [] | |
| return spans | |
| def number_accuracy(refs, hyps): | |
| """Share of the reference's numbers found whole in the transcript ("de mil vennkat" as "de mil kat" is wrong).""" | |
| hit = total = 0 | |
| for r, h in zip(refs, hyps): | |
| hyp = " " + " ".join(mform(h).split()) + " " | |
| for span in number_spans(r): | |
| total += 1 | |
| hit += f" {span} " in hyp | |
| return (hit / total if total else None), total | |
| def diff_html(ref, hyp): | |
| """The hypothesis as scored, word by word: missing words struck through in red, extra or wrong ones in orange.""" | |
| r, h = mform(ref).split(), mform(hyp).split() | |
| out = [] | |
| for op, a, b, c, d in difflib.SequenceMatcher(a=r, b=h, autojunk=False).get_opcodes(): | |
| if op == "equal": | |
| out.append(html.escape(" ".join(h[c:d]))) | |
| continue | |
| if b > a: | |
| out.append(f'<del class="miss">{html.escape(" ".join(r[a:b]))}</del>') | |
| if d > c: | |
| out.append(f'<ins class="extra">{html.escape(" ".join(h[c:d]))}</ins>') | |
| return " ".join(out) | |