File size: 2,829 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | """Thesis-B gold set (line images of 113 inscription lines, Perfilev 2026): CER with spaces removed, as in the
thesis, for our recognizers and for the released thesis adapters run by us (perfilev_infer.py). Lines whose
inscription is in our training split are excluded from the main comparison (37 of 112 remain); scores on all lines
are reported as well. Also scores the thesis's own released predictions (qwen25vl-7b_finetuned_gold.csv) to check
that our run of the adapter reproduces them. Writes results/gold_eval.json."""
import csv
import glob
import json
import os
import random
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import metrics # noqa: E402
from loopcut import cut_loops # noqa: E402
ROOT = os.path.expanduser("~/runic_access")
B = 1000
def nospace(s):
return "".join((s or "").split())
def score(rows):
parts = [metrics.cer_parts(nospace(r["target"]), nospace(r["pred"])) for r in rows]
insc = {}
for i, r in enumerate(rows): # one cluster per inscription: gold_<a>_<b>_<k>str
insc.setdefault(r["id"].rsplit("_", 1)[0], []).append(i)
groups = list(insc.values())
rng = random.Random(0)
def agg(idx):
return 100 * sum(parts[i][0] for i in idx) / max(sum(parts[i][1] for i in idx), 1)
boots = sorted(agg([i for _ in groups for i in rng.choice(groups)]) for _ in range(B))
exact = 100 * sum(nospace(r["target"]) == nospace(r["pred"]) for r in rows) / len(rows)
return dict(n=len(rows), cer=agg(range(len(rows))), cer_ci=(boots[int(0.025 * B)], boots[int(0.975 * B)]), exact=exact)
out = {}
for path in sorted(glob.glob(f"{ROOT}/preds/ocr_gold_*.jsonl")):
tag = os.path.basename(path)[9:-6]
rows = [json.loads(l) for l in open(path)]
rows = [dict(r, pred=cut_loops(r.get("pred_raw", r["pred"]) or "")) for r in rows]
out[tag] = {"nontrain": score([r for r in rows if r["ours"] != "train"]), "all": score(rows),
"unseen": score([r for r in rows if r["ours"] == "unseen"])}
# the thesis's own predictions, matched by file name
ref = {json.loads(l)["image"].split("/")[-1]: json.loads(l) for l in open(glob.glob(f"{ROOT}/preds/ocr_gold_*.jsonl")[0])}
th = [dict(ref[r["file"]], pred=r["pred"]) for r in csv.DictReader(open(f"{ROOT}/sources/repos/RUNIC-OCR/results/predictions/qwen25vl-7b_finetuned_gold.csv")) if r["file"] in ref]
out["thesis_csv_qwen25vl7b"] = {"nontrain": score([r for r in th if r["ours"] != "train"]), "all": score(th)}
for k, v in out.items():
print(f"{k:28s} nontrain n={v['nontrain']['n']} CER {v['nontrain']['cer']:.1f} [{v['nontrain']['cer_ci'][0]:.1f},{v['nontrain']['cer_ci'][1]:.1f}]"
f" exact {v['nontrain']['exact']:.1f} | all n={v['all']['n']} CER {v['all']['cer']:.1f}")
json.dump(out, open(f"{ROOT}/results/gold_eval.json", "w"), indent=1)
|