| """Other backbones trained with our recipes (Reviewer 1, comment 3): translators (Qwen3-4B-Instruct-2507, |
| Ministral-8B-Instruct-2410, Olmo-3-7B-Instruct) against the Qwen3.5-4B translator, and recognizers (Qwen2.5-VL-7B, |
| Qwen3-VL-8B, crop recipe) against the Qwen3.5-4B crop recognizer; plus the thesis-B adapters run on our test images. |
| Missing prediction files are skipped. Writes results/backbones.json.""" |
| import json |
| import os |
| import statistics |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| import metrics |
| from analyze import load, domain, rare_word_recall, name_recall |
| from loopcut import cut_loops |
|
|
| ROOT = os.path.expanduser("~/runic_access") |
| out = {"mt": {}, "ocr": {}, "perfilev": {}} |
|
|
| |
| MT = [("mt4b", "Qwen/Qwen3.5-4B"), ("q3i4b", "Qwen/Qwen3-4B-Instruct-2507"), |
| ("ministral8b", "mistralai/Ministral-8B-Instruct-2410"), ("olmo3_7b", "allenai/Olmo-3-7B-Instruct")] |
| base = {"ref": load("mt_test_mt4b"), "ocr": load("e2e_ocr4b_cropcut_mt4b_fromocr")} |
| for tag, hf in MT: |
| ref = load(f"mt_test_{tag}") |
| ocr = load(f"e2e_ocr4b_cropcut_{tag}_fromocr") |
| if not ref: |
| continue |
| d = dict(hf=hf, ref=metrics.mt_scores(ref), en=metrics.mt_scores([r for r in ref if r["lang"] == "en"], boot=False), |
| sv=metrics.mt_scores([r for r in ref if r["lang"] == "sv"], boot=False), names=name_recall(ref)) |
| zs = load(f"mt_test_{tag}_zs") |
| if zs: |
| d["zero_shot"] = metrics.mt_scores(zs, boot=False) |
| if ocr: |
| d["ocr"] = metrics.mt_scores(ocr) |
| d["ocr_names"] = name_recall(ocr) |
| if tag != "mt4b": |
| d["paired_mt4b_minus_this_ref"] = metrics.paired_bootstrap_chrf(base["ref"], ref) |
| d["paired_this_minus_mt4b_ref"] = metrics.paired_bootstrap_chrf(ref, base["ref"]) |
| if ocr: |
| d["paired_mt4b_minus_this_ocr"] = metrics.paired_bootstrap_chrf(base["ocr"], ocr) |
| d["paired_this_minus_mt4b_ocr"] = metrics.paired_bootstrap_chrf(ocr, base["ocr"]) |
| out["mt"][tag] = d |
| print(f"MT {tag:12s} ref {d['ref']['chrfpp']:.1f} [{d['ref']['chrfpp_ci'][0]:.1f},{d['ref']['chrfpp_ci'][1]:.1f}] " |
| f"en {d['en']['chrfpp']:.1f} sv {d['sv']['chrfpp']:.1f} names {d['names']:.1f}" |
| + (f" zs {d['zero_shot']['chrfpp']:.1f}" if "zero_shot" in d else "") |
| + (f" | photo {d['ocr']['chrfpp']:.1f} names {d['ocr_names']:.1f}" if "ocr" in d else "") |
| + (f" | p(mt4b>this) ref {d['paired_mt4b_minus_this_ref']['p']:.3f} p(this>mt4b) ref {d['paired_this_minus_mt4b_ref']['p']:.3f}" if tag != "mt4b" else "")) |
|
|
|
|
| |
| def ocr_block(rows, e2e=None): |
| rows = [dict(r, pred=cut_loops(r.get("pred_raw", r["pred"]) or "")) for r in rows] |
| cers = [metrics.cer_parts(r["target"], r["pred"]) for r in rows] |
| d = dict(all=metrics.ocr_scores(rows), median_cer=statistics.median(100 * e / max(n, 1) for e, n in cers), |
| rare_recall=rare_word_recall(rows), by_domain={}) |
| for dom in ("gr", "rdb_photo", "card"): |
| sub = [r for r in rows if domain(r) == dom] |
| if sub: |
| d["by_domain"][dom] = metrics.ocr_scores(sub)["cer"] |
| if e2e: |
| d["e2e"] = metrics.mt_scores(e2e) |
| return d |
|
|
|
|
| OCR = [("ocr4b", "Qwen/Qwen3.5-4B"), ("q25vl7b", "Qwen/Qwen2.5-VL-7B-Instruct"), ("q3vl8b", "Qwen/Qwen3-VL-8B-Instruct")] |
| for tag, hf in OCR: |
| test = load(f"ocr_test_{tag}_crop") if tag != "ocr4b" else load("ocr_test_ocr4b_crop") |
| if not test: |
| continue |
| d = dict(hf=hf, test=ocr_block(test, load(f"e2e_{tag}_cropcut_mt4b_fromocr"))) |
| val = load(f"ocr_val_{tag}_crop") |
| if val: |
| d["val"] = ocr_block(val, load(f"e2eval_{tag}_cropcut_mt4b_fromocr")) |
| syn = load(f"ocr_synval_{tag}_crop") |
| if syn: |
| d["synval_cer"] = metrics.ocr_scores([dict(r, pred=cut_loops(r["pred"])) for r in syn])["cer"] |
| if tag != "ocr4b" and "e2e" in d["test"]: |
| d["paired_ocr4b_minus_this"] = metrics.paired_bootstrap_chrf(load("e2e_ocr4b_cropcut_mt4b_fromocr"), load(f"e2e_{tag}_cropcut_mt4b_fromocr")) |
| d["paired_this_minus_ocr4b"] = metrics.paired_bootstrap_chrf(load(f"e2e_{tag}_cropcut_mt4b_fromocr"), load("e2e_ocr4b_cropcut_mt4b_fromocr")) |
| out["ocr"][tag] = d |
| t = d["test"] |
| print(f"OCR {tag:8s} CER {t['all']['cer']:.1f} med {t['median_cer']:.1f} rare {t['rare_recall']:.1f} " |
| + " ".join(f"{k}:{v:.1f}" for k, v in t["by_domain"].items()) |
| + (f" e2e {t['e2e']['chrfpp']:.1f}" if "e2e" in t else "")) |
|
|
| |
| for t in ("qwen25vl7b", "qwen3vl8b"): |
| for split in ("test", "testcrop", "valcrop", "testcrop320"): |
| rows = load(f"ocr_{split}_perf_{t}") |
| if rows: |
| out["perfilev"][f"{t}_{split}"] = ocr_block(rows) |
| b = out["perfilev"][f"{t}_{split}"] |
| print(f"PERF {t} {split:12s} n={b['all']['n']} CER {b['all']['cer']:.1f} med {b['median_cer']:.1f} rare {b['rare_recall']:.1f}") |
| json.dump(out, open(f"{ROOT}/results/backbones.json", "w"), indent=1) |
|
|
