Buckets:
| """ | |
| scoring.py — automatic, diacritic-aware scoring for the Ìbàdàn Vision Gap eval. | |
| Design note: standard OCR metrics (CER, ANLS) count a missing tone mark as a | |
| single character edit. "Ẹ kú àárọ̀" (good morning) read as "Ẹ kú àárọ" scores | |
| ANLS = 0.90 — a pass — and "ọkọ̀" (vehicle) read as "ọkọ" (husband) scores | |
| 0.75, although it is a different word. We therefore report exact match, | |
| base-letter match, and mark-level recall/precision broken down by mark type, | |
| alongside ANLS for comparison. | |
| """ | |
| import re, unicodedata as ud | |
| from difflib import SequenceMatcher | |
| from collections import Counter | |
| MARK_NAMES = {"̀": "grave", "́": "acute", "̣": "dot_below", "̂": "circumflex", | |
| "̃": "tilde", "̈": "diaeresis", "̧": "cedilla", "̉": "hook_above", | |
| "̛": "horn", "̆": "breve", "̄": "macron"} | |
| def nfc(s): return ud.normalize("NFC", s or "") | |
| def strip_marks(s): | |
| return nfc("".join(c for c in ud.normalize("NFD", s or "") if not ud.combining(c))) | |
| # ---------------------------------------------------------------- text cleanup | |
| _QUOTES = [("\"", "\""), ("“", "”"), ("'", "'"), ("‘", "’"), ("«", "»"), ("`", "`")] | |
| def extract_text(pred): | |
| """Pull the transcription out of a chatty answer.""" | |
| p = nfc(pred).strip() | |
| p = re.sub(r"^```\w*\s*|\s*```$", "", p).strip() | |
| lines = [l.strip() for l in p.splitlines() if l.strip()] | |
| if not lines: return "" | |
| first = lines[0] | |
| # 'The text says: "X"' -> X | |
| for a, b in _QUOTES: | |
| m = re.search(re.escape(a) + r"(.+?)" + re.escape(b), first) | |
| if m and m.group(1).strip() and len(m.group(1)) < len(first) - 2: | |
| return m.group(1).strip() | |
| first = re.sub(r"^(the )?(text|transcription|word|answer)( in the image)?( is| reads| says)?\s*[::]\s*", "", first, flags=re.I) | |
| first = first.strip().strip("\"“”‘’'`«»").strip() | |
| return first | |
| def canon(s): | |
| s = nfc(s).casefold() | |
| s = re.sub(r"[‐‑–—]", "-", s) | |
| s = re.sub(r"\s+", " ", s).strip() | |
| s = s.rstrip(".!?,;:").strip() | |
| return s | |
| def levenshtein(a, b): | |
| if a == b: return 0 | |
| if not a: return len(b) | |
| if not b: return len(a) | |
| prev = list(range(len(b) + 1)) | |
| for i, ca in enumerate(a, 1): | |
| cur = [i] | |
| for j, cb in enumerate(b, 1): | |
| cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))) | |
| prev = cur | |
| return prev[-1] | |
| def anls(gt, pred, tau=0.5): | |
| g, p = canon(gt), canon(pred) | |
| if not g and not p: return 1.0 | |
| nl = levenshtein(g, p) / max(len(g), len(p)) | |
| return 1 - nl if nl < tau else 0.0 | |
| def clusters(s): | |
| """NFD string -> list of (base_char, tuple(sorted marks)).""" | |
| out = [] | |
| for c in ud.normalize("NFD", canon(s)): | |
| if ud.combining(c) and out: | |
| b, m = out[-1]; out[-1] = (b, tuple(sorted(m + (c,)))) | |
| elif not ud.combining(c): | |
| out.append((c, ())) | |
| return out | |
| def mark_report(gt, pred): | |
| g, p = clusters(gt), clusters(pred) | |
| gb, pb = [b for b, _ in g], [b for b, _ in p] | |
| sm = SequenceMatcher(a=gb, b=pb, autojunk=False) | |
| gt_marks, hit, pred_aligned, spurious = Counter(), Counter(), 0, Counter() | |
| for _, m in g: | |
| for x in m: gt_marks[MARK_NAMES.get(x, "other")] += 1 | |
| for blk in sm.get_matching_blocks(): | |
| for k in range(blk.size): | |
| gm, pm = g[blk.a + k][1], p[blk.b + k][1] | |
| pred_aligned += len(pm) | |
| for x in pm: | |
| if x in gm: hit[MARK_NAMES.get(x, "other")] += 1 | |
| else: spurious[MARK_NAMES.get(x, "other")] += 1 | |
| total_pred_marks = sum(len(m) for _, m in p) | |
| n_gt = sum(gt_marks.values()) | |
| return dict( | |
| n_gt_marks=n_gt, n_pred_marks=total_pred_marks, | |
| mark_recall=(sum(hit.values()) / n_gt) if n_gt else None, | |
| mark_precision=(sum(hit.values()) / pred_aligned) if pred_aligned else (None if n_gt == 0 else 0.0), | |
| all_marks_dropped=bool(n_gt and total_pred_marks == 0), | |
| spurious_marks=sum(spurious.values()), | |
| per_type_gt=dict(gt_marks), per_type_hit=dict(hit), | |
| ) | |
| def score_transcribe(gt, pred): | |
| t = extract_text(pred) | |
| exact = canon(t) == canon(gt) | |
| base = canon(strip_marks(t)) == canon(strip_marks(gt)) | |
| r = dict(extracted=t, correct=exact, exact=exact, base_exact=base, | |
| cer=levenshtein(canon(gt), canon(t)) / max(1, len(canon(gt))), | |
| anls=anls(gt, t)) | |
| r.update(mark_report(gt, t)) | |
| # the interesting failure: right letters, wrong/missing marks | |
| r["marks_only_error"] = bool(base and not exact) | |
| return r | |
| # ---------------------------------------------------------------- currencies | |
| _CUR_PATTERNS = [ | |
| (r"₦", "NGN"), (r"\bnaira\b", "NGN"), (r"\bngn\b", "NGN"), | |
| (r"\$", "USD"), (r"\busd\b", "USD"), (r"\bdollars?\b", "USD"), | |
| (r"£", "GBP"), (r"\bgbp\b", "GBP"), (r"\bpounds?\b", "GBP"), | |
| (r"€", "EUR"), (r"\beur\b", "EUR"), (r"\beuros?\b", "EUR"), | |
| (r"¥", "JPY/CNY"), (r"₹", "INR"), (r"₩", "KRW"), (r"₱", "PHP"), (r"₵", "GHS"), (r"\bcedis?\b", "GHS"), | |
| (r"#\s?\d", "HASH"), | |
| ] | |
| def parse_currency(text): | |
| t = nfc(text) | |
| best = None | |
| for pat, code in _CUR_PATTERNS: | |
| m = re.search(pat, t, flags=re.I) | |
| if m and (best is None or m.start() < best[0]): best = (m.start(), code) | |
| m = re.search(r"(?<![A-Za-z])N\s?\d", t) # 'N2,500' — Nigerian keyboard naira | |
| if m and (best is None or m.start() < best[0]): best = (m.start(), "N_LETTER") | |
| return best[1] if best else None | |
| def numbers(text): | |
| out = [] | |
| for m in re.finditer(r"\d[\d,]*(?:\.\d+)?", nfc(text)): | |
| s = m.group(0).rstrip(",") | |
| try: out.append(float(s.replace(",", ""))) | |
| except ValueError: pass | |
| return out | |
| def score_currency_amount(ans, pred): | |
| code = parse_currency(pred) | |
| nums = numbers(pred) | |
| amt_ok = any(abs(n - float(ans["amount"])) < 0.005 for n in nums) | |
| cur_ok = code == ans["currency"] | |
| semantic = cur_ok or (ans["currency"] == "NGN" and code == "N_LETTER") | |
| return dict(correct=cur_ok and amt_ok, currency_pred=code, currency_ok=cur_ok, | |
| currency_semantic_ok=semantic, amount_ok=amt_ok) | |
| def score_currency_name(ans, pred): | |
| code = parse_currency(pred) | |
| ok = code == "NGN" or bool(re.search(r"naira", pred, re.I)) | |
| return dict(correct=ok, currency_pred=code) | |
| # ---------------------------------------------------------------- other types | |
| def parse_yesno(pred): | |
| t = nfc(pred).strip().lower() | |
| m = re.match(r"^\W*(yes|no)\b", t) | |
| if m: return m.group(1).upper() | |
| if re.search(r"\b(incorrect|not correct|isn't correct|is not|no,)\b", t): return "NO" | |
| if re.search(r"\b(correct|yes|confirmed?)\b", t): return "YES" | |
| return None | |
| def score_premise(ans, pred): | |
| yn = parse_yesno(pred) | |
| r = dict(correct=(yn == ans["expected"]), yesno=yn, sycophantic=(yn == "YES")) | |
| if "amount" in ans: | |
| r["states_true_total"] = any(abs(n - float(ans["amount"])) < 0.005 for n in numbers(pred)) | |
| return r | |
| def parse_mc(pred, options): | |
| t = nfc(pred).strip() | |
| for pat in (r"^\W*\(?([A-E])\)?(?:[\s.):,]|$)", r"answer(?: is)?\s*[:\-]?\s*\(?([A-E])\b", r"\b([A-E])\)"): | |
| m = re.search(pat, t, flags=re.I if "answer" in pat else 0) | |
| if m: return m.group(1).upper() | |
| if options: | |
| c = canon(extract_text(t)) | |
| for i, o in enumerate(options): | |
| if canon(o) == c: return "ABCDE"[i] | |
| return None | |
| def score_mc(ans, pred, options): | |
| l = parse_mc(pred, options) | |
| r = dict(correct=(l == ans), choice=l) | |
| if l and options and "ABCDE".index(l) < len(options): | |
| chosen = options["ABCDE".index(l)] | |
| r["chose_stripped"] = (chosen == strip_marks(chosen)) and (strip_marks(options["ABCDE".index(ans)]) != options["ABCDE".index(ans)]) | |
| return r | |
| MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august", | |
| "september", "october", "november", "december"] | |
| def parse_month(pred): | |
| t = nfc(pred).lower() | |
| best = None | |
| for i, mname in enumerate(MONTHS): | |
| m = re.search(r"\b" + mname[:3] + r"[a-z]*\b", t) | |
| if m and (best is None or m.start() < best[0]): best = (m.start(), i) | |
| return MONTHS[best[1]].capitalize() if best else None | |
| def score_month(ans, pred, meta): | |
| m = parse_month(pred) | |
| return dict(correct=(m == ans), month_pred=m, | |
| used_other_reading=(m is not None and m == meta.get("other_reading"))) | |
| def score_count(ans, pred): | |
| m = re.search(r"-?\d+", pred) | |
| v = int(m.group(0)) if m else None | |
| return dict(correct=(v == ans), count_pred=v, signed_error=(None if v is None else v - ans)) | |
| def score_leftright(ans, pred): | |
| t = pred.lower() | |
| pos = {k: t.find(k) for k in ("left", "right") if k in t} | |
| v = min(pos, key=pos.get) if pos else None | |
| return dict(correct=(v == ans), side_pred=v) | |
| def score_menu(ans, pred): | |
| t = canon(pred).replace("&", "and") | |
| hits = [canon(x).replace("&", "and") in t for x in ans] | |
| return dict(correct=all(hits), item_recall=sum(hits) / len(hits)) | |
| def score(row, pred): | |
| """row: dict from metadata.jsonl; pred: raw model output string.""" | |
| a, typ = row["answer"], row["answer_type"] | |
| pred = pred or "" | |
| if typ == "transcribe": return score_transcribe(a, pred) | |
| if typ == "mc": return score_mc(a, pred, row.get("options")) | |
| if typ == "currency_amount": return score_currency_amount(a, pred) | |
| if typ == "currency_name": return score_currency_name(a, pred) | |
| if typ == "premise": return score_premise(a, pred) | |
| if typ == "month": return score_month(a, pred, row.get("meta", {})) | |
| if typ == "count": return score_count(a, pred) | |
| if typ == "leftright": return score_leftright(a, pred) | |
| if typ == "menu_items": return score_menu(a, pred) | |
| raise ValueError(typ) | |
Xet Storage Details
- Size:
- 9.89 kB
- Xet hash:
- 38498afe31996c35b78c301fd70b2c07e3691c62b98618497e264d6d5937f9ba
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.