| """Thesis-B gold set (line images of 113 inscription lines, Perfilev 2026): CER with spaces removed, as in the |
| thesis, for our recognizers and for the released thesis adapters run by us (perfilev_infer.py). Lines whose |
| inscription is in our training split are excluded from the main comparison (37 of 112 remain); scores on all lines |
| are reported as well. Also scores the thesis's own released predictions (qwen25vl-7b_finetuned_gold.csv) to check |
| that our run of the adapter reproduces them. Writes results/gold_eval.json.""" |
| import csv |
| import glob |
| import json |
| import os |
| import random |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| import metrics |
| from loopcut import cut_loops |
|
|
| ROOT = os.path.expanduser("~/runic_access") |
| B = 1000 |
|
|
|
|
| def nospace(s): |
| return "".join((s or "").split()) |
|
|
|
|
| def score(rows): |
| parts = [metrics.cer_parts(nospace(r["target"]), nospace(r["pred"])) for r in rows] |
| insc = {} |
| for i, r in enumerate(rows): |
| insc.setdefault(r["id"].rsplit("_", 1)[0], []).append(i) |
| groups = list(insc.values()) |
| rng = random.Random(0) |
|
|
| def agg(idx): |
| return 100 * sum(parts[i][0] for i in idx) / max(sum(parts[i][1] for i in idx), 1) |
| boots = sorted(agg([i for _ in groups for i in rng.choice(groups)]) for _ in range(B)) |
| exact = 100 * sum(nospace(r["target"]) == nospace(r["pred"]) for r in rows) / len(rows) |
| return dict(n=len(rows), cer=agg(range(len(rows))), cer_ci=(boots[int(0.025 * B)], boots[int(0.975 * B)]), exact=exact) |
|
|
|
|
| out = {} |
| for path in sorted(glob.glob(f"{ROOT}/preds/ocr_gold_*.jsonl")): |
| tag = os.path.basename(path)[9:-6] |
| rows = [json.loads(l) for l in open(path)] |
| rows = [dict(r, pred=cut_loops(r.get("pred_raw", r["pred"]) or "")) for r in rows] |
| out[tag] = {"nontrain": score([r for r in rows if r["ours"] != "train"]), "all": score(rows), |
| "unseen": score([r for r in rows if r["ours"] == "unseen"])} |
| |
| ref = {json.loads(l)["image"].split("/")[-1]: json.loads(l) for l in open(glob.glob(f"{ROOT}/preds/ocr_gold_*.jsonl")[0])} |
| th = [dict(ref[r["file"]], pred=r["pred"]) for r in csv.DictReader(open(f"{ROOT}/sources/repos/RUNIC-OCR/results/predictions/qwen25vl-7b_finetuned_gold.csv")) if r["file"] in ref] |
| out["thesis_csv_qwen25vl7b"] = {"nontrain": score([r for r in th if r["ours"] != "train"]), "all": score(th)} |
| for k, v in out.items(): |
| print(f"{k:28s} nontrain n={v['nontrain']['n']} CER {v['nontrain']['cer']:.1f} [{v['nontrain']['cer_ci'][0]:.1f},{v['nontrain']['cer_ci'][1]:.1f}]" |
| f" exact {v['nontrain']['exact']:.1f} | all n={v['all']['n']} CER {v['all']['cer']:.1f}") |
| json.dump(out, open(f"{ROOT}/results/gold_eval.json", "w"), indent=1) |
|
|
