| """Assemble the data/code release promised in the Access paper's Data and Code Availability section. |
| |
| Writes release_access/ (published as huggingface.co/kesha-humonen/runic-reader-2 together with the adapters in release_adapters/): |
| splits/pairs.jsonl the 12,496 transliteration-translation pairs with split and inscription cluster; |
| Rundata and RuneS rows with their texts, Gamla runor rows with identifiers, page and image |
| links and SHA-1 hashes of the normalized transliteration and the translation (no text) |
| splits/images.jsonl the 4,949 real images: record, split, source URL, localization box, test domain |
| preds/*.jsonl test predictions of the main systems; Gamla runor references removed |
| gamla_runor/fetch_gamla_runor.py retrieves the Gamla runor texts and photographs from christerhamp.se |
| code/ corpus construction, training, inference and evaluation scripts |
| """ |
| import hashlib |
| import json |
| import os |
| import shutil |
| import subprocess |
| import sys |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| from runic_norm import normalize |
|
|
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| OUT = f"{ROOT}/release_access" |
| shutil.rmtree(OUT, ignore_errors=True) |
| for d in ("splits", "preds", "gamla_runor", "code/jobs"): |
| os.makedirs(f"{OUT}/{d}", exist_ok=True) |
|
|
|
|
| def sha(s): |
| return hashlib.sha1((s or "").encode("utf-8")).hexdigest() |
|
|
|
|
| def rd(p): |
| return [json.loads(l) for l in open(p) if l.strip()] |
|
|
|
|
| corpus = {r["id"]: r for r in rd(f"{ROOT}/data/corpus.jsonl")} |
| lf = json.load(open(f"{ROOT}/data/leakfree.json")) |
| keep = {"train": None, "val": set(lf["keep_val"]), "test": set(lf["keep_test"])} |
|
|
| |
| n = 0 |
| with open(f"{OUT}/splits/pairs.jsonl", "w") as f: |
| for split in ("train", "val", "test"): |
| for r in rd(f"{ROOT}/splits_local/mt_{split}.jsonl"): |
| if keep[split] is not None and r["id"] not in keep[split]: |
| continue |
| c = corpus[r["id"]] |
| rec = dict(id=r["id"], source={"srd": "rundata", "rdb": "runes", "ch": "gamla_runor"}[c["src"]], |
| signum=c.get("signum"), split=split, cluster=int(c["cluster"]), lang=c["lang"], page=c.get("page")) |
| if c["src"] == "ch": |
| rec.update(image_urls=c.get("image_urls"), sha1_transliteration=sha(r["src"]), sha1_translation=sha(r["tgt"])) |
| else: |
| rec.update(transliteration_raw=c["translit_raw"], transliteration=r["src"], translation=r["tgt"]) |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") |
| n += 1 |
| print("pairs", n) |
|
|
| |
| boxes = {r["image"]: r["crop_box"] for r in rd(f"{ROOT}/data/crop_boxes.jsonl")} |
| manifest = {m["file"]: m for m in rd(f"{ROOT}/data/image_manifest.jsonl")} |
| card = json.load(open(f"{ROOT}/data/card_labels.json"))["labels"] |
| cream = json.load(open(f"{ROOT}/data/cream_share.json")) |
|
|
|
|
| def domain(r): |
| if r["source"] == "ch": |
| return "gamla_runor_photo" |
| img = r["image"] |
| if img in card: |
| return "runes_photo" if card[img] in ("photo", "manuscript") else "runes_card" |
| return "runes_card" if cream.get(img, 0) > 0.2 else "runes_photo" |
|
|
|
|
| n = 0 |
| with open(f"{OUT}/splits/images.jsonl", "w") as f: |
| for split in ("train", "val", "test"): |
| for r in rd(f"{ROOT}/splits_server/ocr_real_{split}.jsonl"): |
| if keep[split] is not None and r["row_id"] not in keep[split]: |
| continue |
| m = manifest.get(os.path.basename(r["image"]), {}) |
| rec = dict(id=r["id"], record=r["row_id"], split=split, cluster=r["cluster"], url=m.get("url"), |
| crop_box_0_1000=boxes.get(r["image"]), domain=domain(r)) |
| f.write(json.dumps(rec, ensure_ascii=False) + "\n") |
| n += 1 |
| print("images", n) |
|
|
| |
| PRED = ["ocr_test_ocr4b_cropcut", "ocr_test_ocr4b_mixcut", "e2e_ocr4b_cropcut_mt4b_fromocr", "e2e_ocr4b_cropcut_mt4b_noise_fromocr", |
| "e2e_ocr4b_mixcut_mt4b_fromgold", "mt_test_mt4b", "mt_test_mt4b_noise", "mt_test_mt08", "mt_test_retrieval"] |
| DROP = ("target", "tgt", "src", "src_clean", "translit") |
| for p in PRED: |
| rows = rd(f"{ROOT}/preds/{p}.jsonl") |
| with open(f"{OUT}/preds/{p}.jsonl", "w") as f: |
| for r in rows: |
| rid = r.get("row_id") or r["id"] |
| if rid not in keep["test"]: |
| continue |
| if rid.startswith("ch"): |
| r = {k: v for k, v in r.items() if k not in DROP} |
| if p.startswith("mt_test") or p.endswith("fromgold"): |
| r = {k: v for k, v in r.items() if k != "pred"} | {"pred_sha1": sha(r.get("pred"))} |
| f.write(json.dumps(r, ensure_ascii=False) + "\n") |
| print("preds", len(PRED)) |
|
|
| |
| CODE = ["runic_norm.py", "build_corpus.py", "relink.py", "make_splits.py", "doc_filter.py", "red_share.py", "synth.py", |
| "gen_diffusion.py", "ground.py", "make_crops.py", "qwen_common.py", "train_lora.py", "generate.py", "ocr_noise.py", |
| "noise_curve.py", "retrieval_baseline.py", "prior_baselines.py", "loopcut.py", "cutpreds.py", "score_ocr.py", |
| "metrics.py", "analyze.py", "make_tables.py", "make_tables_rev.py", "crop_mechanism.py", "matched_noise.py", |
| "names_e2e.py", "label_sensitivity.py", "perfilev_infer.py", "valsel.py", "backbones.py", "gold_eval.py", |
| "fig_noise.py", "build_release.py"] |
| for c in CODE: |
| if os.path.exists(f"{ROOT}/code/{c}"): |
| shutil.copy(f"{ROOT}/code/{c}", f"{OUT}/code/{c}") |
| else: |
| print("missing code file", c) |
| for j in ("exp_v2.sh", "exp_v4.sh", "exp_v6.sh", "exp_v8.sh", "exp_v9.sh", "common.sh"): |
| shutil.copy(f"{ROOT}/code/jobs/{j}", f"{OUT}/code/jobs/{j}") |
| shutil.copy(f"{ROOT}/release_access_extra/fetch_gamla_runor.py", f"{OUT}/gamla_runor/fetch_gamla_runor.py") |
| leak = subprocess.run(["grep", "-rlE", r"hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9]{20,}|BEGIN (RSA|OPENSSH) PRIVATE", OUT], |
| capture_output=True, text=True).stdout |
| print("token scan:", leak or "clean") |
|
|
