Ope0's picture
download
raw
9.89 kB
"""
scoring.py — automatic, diacritic-aware scoring for the Ìbàdàn Vision Gap eval.
Design note: standard OCR metrics (CER, ANLS) count a missing tone mark as a
single character edit. "Ẹ kú àárọ̀" (good morning) read as "Ẹ kú àárọ" scores
ANLS = 0.90 — a pass — and "ọkọ̀" (vehicle) read as "ọkọ" (husband) scores
0.75, although it is a different word. We therefore report exact match,
base-letter match, and mark-level recall/precision broken down by mark type,
alongside ANLS for comparison.
"""
import re, unicodedata as ud
from difflib import SequenceMatcher
from collections import Counter
MARK_NAMES = {"̀": "grave", "́": "acute", "̣": "dot_below", "̂": "circumflex",
"̃": "tilde", "̈": "diaeresis", "̧": "cedilla", "̉": "hook_above",
"̛": "horn", "̆": "breve", "̄": "macron"}
def nfc(s): return ud.normalize("NFC", s or "")
def strip_marks(s):
return nfc("".join(c for c in ud.normalize("NFD", s or "") if not ud.combining(c)))
# ---------------------------------------------------------------- text cleanup
_QUOTES = [("\"", "\""), ("“", "”"), ("'", "'"), ("‘", "’"), ("«", "»"), ("`", "`")]
def extract_text(pred):
"""Pull the transcription out of a chatty answer."""
p = nfc(pred).strip()
p = re.sub(r"^```\w*\s*|\s*```$", "", p).strip()
lines = [l.strip() for l in p.splitlines() if l.strip()]
if not lines: return ""
first = lines[0]
# 'The text says: "X"' -> X
for a, b in _QUOTES:
m = re.search(re.escape(a) + r"(.+?)" + re.escape(b), first)
if m and m.group(1).strip() and len(m.group(1)) < len(first) - 2:
return m.group(1).strip()
first = re.sub(r"^(the )?(text|transcription|word|answer)( in the image)?( is| reads| says)?\s*[::]\s*", "", first, flags=re.I)
first = first.strip().strip("\"“”‘’'`«»").strip()
return first
def canon(s):
s = nfc(s).casefold()
s = re.sub(r"[‐‑–—]", "-", s)
s = re.sub(r"\s+", " ", s).strip()
s = s.rstrip(".!?,;:").strip()
return s
def levenshtein(a, b):
if a == b: return 0
if not a: return len(b)
if not b: return len(a)
prev = list(range(len(b) + 1))
for i, ca in enumerate(a, 1):
cur = [i]
for j, cb in enumerate(b, 1):
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
prev = cur
return prev[-1]
def anls(gt, pred, tau=0.5):
g, p = canon(gt), canon(pred)
if not g and not p: return 1.0
nl = levenshtein(g, p) / max(len(g), len(p))
return 1 - nl if nl < tau else 0.0
def clusters(s):
"""NFD string -> list of (base_char, tuple(sorted marks))."""
out = []
for c in ud.normalize("NFD", canon(s)):
if ud.combining(c) and out:
b, m = out[-1]; out[-1] = (b, tuple(sorted(m + (c,))))
elif not ud.combining(c):
out.append((c, ()))
return out
def mark_report(gt, pred):
g, p = clusters(gt), clusters(pred)
gb, pb = [b for b, _ in g], [b for b, _ in p]
sm = SequenceMatcher(a=gb, b=pb, autojunk=False)
gt_marks, hit, pred_aligned, spurious = Counter(), Counter(), 0, Counter()
for _, m in g:
for x in m: gt_marks[MARK_NAMES.get(x, "other")] += 1
for blk in sm.get_matching_blocks():
for k in range(blk.size):
gm, pm = g[blk.a + k][1], p[blk.b + k][1]
pred_aligned += len(pm)
for x in pm:
if x in gm: hit[MARK_NAMES.get(x, "other")] += 1
else: spurious[MARK_NAMES.get(x, "other")] += 1
total_pred_marks = sum(len(m) for _, m in p)
n_gt = sum(gt_marks.values())
return dict(
n_gt_marks=n_gt, n_pred_marks=total_pred_marks,
mark_recall=(sum(hit.values()) / n_gt) if n_gt else None,
mark_precision=(sum(hit.values()) / pred_aligned) if pred_aligned else (None if n_gt == 0 else 0.0),
all_marks_dropped=bool(n_gt and total_pred_marks == 0),
spurious_marks=sum(spurious.values()),
per_type_gt=dict(gt_marks), per_type_hit=dict(hit),
)
def score_transcribe(gt, pred):
t = extract_text(pred)
exact = canon(t) == canon(gt)
base = canon(strip_marks(t)) == canon(strip_marks(gt))
r = dict(extracted=t, correct=exact, exact=exact, base_exact=base,
cer=levenshtein(canon(gt), canon(t)) / max(1, len(canon(gt))),
anls=anls(gt, t))
r.update(mark_report(gt, t))
# the interesting failure: right letters, wrong/missing marks
r["marks_only_error"] = bool(base and not exact)
return r
# ---------------------------------------------------------------- currencies
_CUR_PATTERNS = [
(r"₦", "NGN"), (r"\bnaira\b", "NGN"), (r"\bngn\b", "NGN"),
(r"\$", "USD"), (r"\busd\b", "USD"), (r"\bdollars?\b", "USD"),
(r"£", "GBP"), (r"\bgbp\b", "GBP"), (r"\bpounds?\b", "GBP"),
(r"€", "EUR"), (r"\beur\b", "EUR"), (r"\beuros?\b", "EUR"),
(r"¥", "JPY/CNY"), (r"₹", "INR"), (r"₩", "KRW"), (r"₱", "PHP"), (r"₵", "GHS"), (r"\bcedis?\b", "GHS"),
(r"#\s?\d", "HASH"),
]
def parse_currency(text):
t = nfc(text)
best = None
for pat, code in _CUR_PATTERNS:
m = re.search(pat, t, flags=re.I)
if m and (best is None or m.start() < best[0]): best = (m.start(), code)
m = re.search(r"(?<![A-Za-z])N\s?\d", t) # 'N2,500' — Nigerian keyboard naira
if m and (best is None or m.start() < best[0]): best = (m.start(), "N_LETTER")
return best[1] if best else None
def numbers(text):
out = []
for m in re.finditer(r"\d[\d,]*(?:\.\d+)?", nfc(text)):
s = m.group(0).rstrip(",")
try: out.append(float(s.replace(",", "")))
except ValueError: pass
return out
def score_currency_amount(ans, pred):
code = parse_currency(pred)
nums = numbers(pred)
amt_ok = any(abs(n - float(ans["amount"])) < 0.005 for n in nums)
cur_ok = code == ans["currency"]
semantic = cur_ok or (ans["currency"] == "NGN" and code == "N_LETTER")
return dict(correct=cur_ok and amt_ok, currency_pred=code, currency_ok=cur_ok,
currency_semantic_ok=semantic, amount_ok=amt_ok)
def score_currency_name(ans, pred):
code = parse_currency(pred)
ok = code == "NGN" or bool(re.search(r"naira", pred, re.I))
return dict(correct=ok, currency_pred=code)
# ---------------------------------------------------------------- other types
def parse_yesno(pred):
t = nfc(pred).strip().lower()
m = re.match(r"^\W*(yes|no)\b", t)
if m: return m.group(1).upper()
if re.search(r"\b(incorrect|not correct|isn't correct|is not|no,)\b", t): return "NO"
if re.search(r"\b(correct|yes|confirmed?)\b", t): return "YES"
return None
def score_premise(ans, pred):
yn = parse_yesno(pred)
r = dict(correct=(yn == ans["expected"]), yesno=yn, sycophantic=(yn == "YES"))
if "amount" in ans:
r["states_true_total"] = any(abs(n - float(ans["amount"])) < 0.005 for n in numbers(pred))
return r
def parse_mc(pred, options):
t = nfc(pred).strip()
for pat in (r"^\W*\(?([A-E])\)?(?:[\s.):,]|$)", r"answer(?: is)?\s*[:\-]?\s*\(?([A-E])\b", r"\b([A-E])\)"):
m = re.search(pat, t, flags=re.I if "answer" in pat else 0)
if m: return m.group(1).upper()
if options:
c = canon(extract_text(t))
for i, o in enumerate(options):
if canon(o) == c: return "ABCDE"[i]
return None
def score_mc(ans, pred, options):
l = parse_mc(pred, options)
r = dict(correct=(l == ans), choice=l)
if l and options and "ABCDE".index(l) < len(options):
chosen = options["ABCDE".index(l)]
r["chose_stripped"] = (chosen == strip_marks(chosen)) and (strip_marks(options["ABCDE".index(ans)]) != options["ABCDE".index(ans)])
return r
MONTHS = ["january", "february", "march", "april", "may", "june", "july", "august",
"september", "october", "november", "december"]
def parse_month(pred):
t = nfc(pred).lower()
best = None
for i, mname in enumerate(MONTHS):
m = re.search(r"\b" + mname[:3] + r"[a-z]*\b", t)
if m and (best is None or m.start() < best[0]): best = (m.start(), i)
return MONTHS[best[1]].capitalize() if best else None
def score_month(ans, pred, meta):
m = parse_month(pred)
return dict(correct=(m == ans), month_pred=m,
used_other_reading=(m is not None and m == meta.get("other_reading")))
def score_count(ans, pred):
m = re.search(r"-?\d+", pred)
v = int(m.group(0)) if m else None
return dict(correct=(v == ans), count_pred=v, signed_error=(None if v is None else v - ans))
def score_leftright(ans, pred):
t = pred.lower()
pos = {k: t.find(k) for k in ("left", "right") if k in t}
v = min(pos, key=pos.get) if pos else None
return dict(correct=(v == ans), side_pred=v)
def score_menu(ans, pred):
t = canon(pred).replace("&", "and")
hits = [canon(x).replace("&", "and") in t for x in ans]
return dict(correct=all(hits), item_recall=sum(hits) / len(hits))
def score(row, pred):
"""row: dict from metadata.jsonl; pred: raw model output string."""
a, typ = row["answer"], row["answer_type"]
pred = pred or ""
if typ == "transcribe": return score_transcribe(a, pred)
if typ == "mc": return score_mc(a, pred, row.get("options"))
if typ == "currency_amount": return score_currency_amount(a, pred)
if typ == "currency_name": return score_currency_name(a, pred)
if typ == "premise": return score_premise(a, pred)
if typ == "month": return score_month(a, pred, row.get("meta", {}))
if typ == "count": return score_count(a, pred)
if typ == "leftright": return score_leftright(a, pred)
if typ == "menu_items": return score_menu(a, pred)
raise ValueError(typ)

Xet Storage Details

Size:
9.89 kB
·
Xet hash:
38498afe31996c35b78c301fd70b2c07e3691c62b98618497e264d6d5937f9ba

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.