| """Recognition against references without editorially restored or uncertain signs (reviewer 3, point 3). |
| |
| Rundata-style transliterations mark signs restored from older records with [ ] and uncertain readings with ( ). |
| The normalized reference keeps their letters. Here the reference is rebuilt from the raw transliteration with the |
| bracketed content removed ([ ] only, and [ ] plus ( )), and CER of the recognizers is recomputed on the test images |
| whose normalized raw transliteration reproduces the stored reference. Writes results/label_sensitivity.json. |
| """ |
| import json |
| import os |
| import re |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| import metrics |
| from runic_norm import normalize |
|
|
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| corpus = {x["id"]: x for x in map(json.loads, open(f"{ROOT}/data/corpus.jsonl"))} |
| KEEP = set(json.load(open(f"{ROOT}/data/leakfree.json"))["keep_test"]) |
| SQ = re.compile(r"\[[^\]]*\]") |
| RO = re.compile(r"\([^)]*\)") |
| res = {} |
| for tag in ("ocr4b_cropcut", "ocr4b_mixcut"): |
| rows = [r for r in metrics.read(f"{ROOT}/preds/ocr_test_{tag}.jsonl") if r["row_id"] in KEEP] |
| ok = [r for r in rows if normalize(corpus[r["row_id"]]["translit_raw"]) == r["target"]] |
| out = {"n_all": len(rows), "n_consistent": len(ok), "full": metrics.ocr_scores(ok)} |
| for name, pats in (("without_square", [SQ]), ("without_square_round", [SQ, RO])): |
| vis = [] |
| for r in ok: |
| raw = corpus[r["row_id"]]["translit_raw"] |
| for p in pats: |
| raw = p.sub(" ", raw) |
| v = normalize(raw) |
| if v: |
| vis.append(dict(r, target=v)) |
| out[name] = metrics.ocr_scores(vis) |
| ids = {x["id"] for x in vis} |
| out[name]["cer_full_same_images"] = metrics.ocr_scores([r for r in ok if r["id"] in ids])["cer"] |
| out[name]["ref_chars_removed_pct"] = 100 * (1 - sum(len(x["target"]) for x in vis) / sum(len(r["target"]) for r in ok if any(r["id"] == x["id"] for x in vis))) |
| res[tag] = out |
| print(tag, out["n_all"], out["n_consistent"], {k: (round(v["cer"], 1), round(v.get("cer_full_same_images", v["cer"]), 1), round(v.get("ref_chars_removed_pct", 0), 1), v["n"]) for k, v in out.items() if isinstance(v, dict)}) |
| json.dump(res, open(f"{ROOT}/results/label_sensitivity.json", "w"), indent=1) |
|
|
