File size: 6,239 Bytes
ed46c5d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Assemble the data/code release promised in the Access paper's Data and Code Availability section.
Writes release_access/ (published as huggingface.co/kesha-humonen/runic-reader-2 together with the adapters in release_adapters/):
splits/pairs.jsonl the 12,496 transliteration-translation pairs with split and inscription cluster;
Rundata and RuneS rows with their texts, Gamla runor rows with identifiers, page and image
links and SHA-1 hashes of the normalized transliteration and the translation (no text)
splits/images.jsonl the 4,949 real images: record, split, source URL, localization box, test domain
preds/*.jsonl test predictions of the main systems; Gamla runor references removed
gamla_runor/fetch_gamla_runor.py retrieves the Gamla runor texts and photographs from christerhamp.se
code/ corpus construction, training, inference and evaluation scripts
"""
import hashlib
import json
import os
import shutil
import subprocess
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from runic_norm import normalize # noqa: E402
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = f"{ROOT}/release_access"
shutil.rmtree(OUT, ignore_errors=True)
for d in ("splits", "preds", "gamla_runor", "code/jobs"):
os.makedirs(f"{OUT}/{d}", exist_ok=True)
def sha(s):
return hashlib.sha1((s or "").encode("utf-8")).hexdigest()
def rd(p):
return [json.loads(l) for l in open(p) if l.strip()]
corpus = {r["id"]: r for r in rd(f"{ROOT}/data/corpus.jsonl")}
lf = json.load(open(f"{ROOT}/data/leakfree.json"))
keep = {"train": None, "val": set(lf["keep_val"]), "test": set(lf["keep_test"])}
# ---------------- pairs ----------------
n = 0
with open(f"{OUT}/splits/pairs.jsonl", "w") as f:
for split in ("train", "val", "test"):
for r in rd(f"{ROOT}/splits_local/mt_{split}.jsonl"):
if keep[split] is not None and r["id"] not in keep[split]:
continue
c = corpus[r["id"]]
rec = dict(id=r["id"], source={"srd": "rundata", "rdb": "runes", "ch": "gamla_runor"}[c["src"]],
signum=c.get("signum"), split=split, cluster=int(c["cluster"]), lang=c["lang"], page=c.get("page"))
if c["src"] == "ch":
rec.update(image_urls=c.get("image_urls"), sha1_transliteration=sha(r["src"]), sha1_translation=sha(r["tgt"]))
else:
rec.update(transliteration_raw=c["translit_raw"], transliteration=r["src"], translation=r["tgt"])
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
n += 1
print("pairs", n)
# ---------------- images ----------------
boxes = {r["image"]: r["crop_box"] for r in rd(f"{ROOT}/data/crop_boxes.jsonl")}
manifest = {m["file"]: m for m in rd(f"{ROOT}/data/image_manifest.jsonl")}
card = json.load(open(f"{ROOT}/data/card_labels.json"))["labels"]
cream = json.load(open(f"{ROOT}/data/cream_share.json"))
def domain(r):
if r["source"] == "ch":
return "gamla_runor_photo"
img = r["image"]
if img in card:
return "runes_photo" if card[img] in ("photo", "manuscript") else "runes_card"
return "runes_card" if cream.get(img, 0) > 0.2 else "runes_photo"
n = 0
with open(f"{OUT}/splits/images.jsonl", "w") as f:
for split in ("train", "val", "test"):
for r in rd(f"{ROOT}/splits_server/ocr_real_{split}.jsonl"):
if keep[split] is not None and r["row_id"] not in keep[split]:
continue
m = manifest.get(os.path.basename(r["image"]), {})
rec = dict(id=r["id"], record=r["row_id"], split=split, cluster=r["cluster"], url=m.get("url"),
crop_box_0_1000=boxes.get(r["image"]), domain=domain(r))
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
n += 1
print("images", n)
# ---------------- predictions (Gamla runor references removed) ----------------
PRED = ["ocr_test_ocr4b_cropcut", "ocr_test_ocr4b_mixcut", "e2e_ocr4b_cropcut_mt4b_fromocr", "e2e_ocr4b_cropcut_mt4b_noise_fromocr",
"e2e_ocr4b_mixcut_mt4b_fromgold", "mt_test_mt4b", "mt_test_mt4b_noise", "mt_test_mt08", "mt_test_retrieval"]
DROP = ("target", "tgt", "src", "src_clean", "translit")
for p in PRED:
rows = rd(f"{ROOT}/preds/{p}.jsonl")
with open(f"{OUT}/preds/{p}.jsonl", "w") as f:
for r in rows:
rid = r.get("row_id") or r["id"]
if rid not in keep["test"]:
continue
if rid.startswith("ch"):
r = {k: v for k, v in r.items() if k not in DROP}
if p.startswith("mt_test") or p.endswith("fromgold"):
r = {k: v for k, v in r.items() if k != "pred"} | {"pred_sha1": sha(r.get("pred"))}
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print("preds", len(PRED))
# ---------------- code ----------------
CODE = ["runic_norm.py", "build_corpus.py", "relink.py", "make_splits.py", "doc_filter.py", "red_share.py", "synth.py",
"gen_diffusion.py", "ground.py", "make_crops.py", "qwen_common.py", "train_lora.py", "generate.py", "ocr_noise.py",
"noise_curve.py", "retrieval_baseline.py", "prior_baselines.py", "loopcut.py", "cutpreds.py", "score_ocr.py",
"metrics.py", "analyze.py", "make_tables.py", "make_tables_rev.py", "crop_mechanism.py", "matched_noise.py",
"names_e2e.py", "label_sensitivity.py", "perfilev_infer.py", "valsel.py", "backbones.py", "gold_eval.py",
"fig_noise.py", "build_release.py"]
for c in CODE:
if os.path.exists(f"{ROOT}/code/{c}"):
shutil.copy(f"{ROOT}/code/{c}", f"{OUT}/code/{c}")
else:
print("missing code file", c)
for j in ("exp_v2.sh", "exp_v4.sh", "exp_v6.sh", "exp_v8.sh", "exp_v9.sh", "common.sh"):
shutil.copy(f"{ROOT}/code/jobs/{j}", f"{OUT}/code/jobs/{j}")
shutil.copy(f"{ROOT}/release_access_extra/fetch_gamla_runor.py", f"{OUT}/gamla_runor/fetch_gamla_runor.py")
leak = subprocess.run(["grep", "-rlE", r"hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9]{20,}|BEGIN (RSA|OPENSSH) PRIVATE", OUT],
capture_output=True, text=True).stdout
print("token scan:", leak or "clean")
|