File size: 2,376 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | """Recognition against references without editorially restored or uncertain signs (reviewer 3, point 3).
Rundata-style transliterations mark signs restored from older records with [ ] and uncertain readings with ( ).
The normalized reference keeps their letters. Here the reference is rebuilt from the raw transliteration with the
bracketed content removed ([ ] only, and [ ] plus ( )), and CER of the recognizers is recomputed on the test images
whose normalized raw transliteration reproduces the stored reference. Writes results/label_sensitivity.json.
"""
import json
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import metrics # noqa: E402
from runic_norm import normalize # noqa: E402
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
corpus = {x["id"]: x for x in map(json.loads, open(f"{ROOT}/data/corpus.jsonl"))}
KEEP = set(json.load(open(f"{ROOT}/data/leakfree.json"))["keep_test"])
SQ = re.compile(r"\[[^\]]*\]")
RO = re.compile(r"\([^)]*\)")
res = {}
for tag in ("ocr4b_cropcut", "ocr4b_mixcut"):
rows = [r for r in metrics.read(f"{ROOT}/preds/ocr_test_{tag}.jsonl") if r["row_id"] in KEEP]
ok = [r for r in rows if normalize(corpus[r["row_id"]]["translit_raw"]) == r["target"]]
out = {"n_all": len(rows), "n_consistent": len(ok), "full": metrics.ocr_scores(ok)}
for name, pats in (("without_square", [SQ]), ("without_square_round", [SQ, RO])):
vis = []
for r in ok:
raw = corpus[r["row_id"]]["translit_raw"]
for p in pats:
raw = p.sub(" ", raw)
v = normalize(raw)
if v:
vis.append(dict(r, target=v))
out[name] = metrics.ocr_scores(vis)
ids = {x["id"] for x in vis}
out[name]["cer_full_same_images"] = metrics.ocr_scores([r for r in ok if r["id"] in ids])["cer"]
out[name]["ref_chars_removed_pct"] = 100 * (1 - sum(len(x["target"]) for x in vis) / sum(len(r["target"]) for r in ok if any(r["id"] == x["id"] for x in vis)))
res[tag] = out
print(tag, out["n_all"], out["n_consistent"], {k: (round(v["cer"], 1), round(v.get("cer_full_same_images", v["cer"]), 1), round(v.get("ref_chars_removed_pct", 0), 1), v["n"]) for k, v in out.items() if isinstance(v, dict)})
json.dump(res, open(f"{ROOT}/results/label_sensitivity.json", "w"), indent=1)
|