"""Assemble the data/code release promised in the Access paper's Data and Code Availability section.

Writes release_access/ (published as huggingface.co/kesha-humonen/runic-reader-2 together with the adapters in release_adapters/):
  splits/pairs.jsonl     the 12,496 transliteration-translation pairs with split and inscription cluster;
                         Rundata and RuneS rows with their texts, Gamla runor rows with identifiers, page and image
                         links and SHA-1 hashes of the normalized transliteration and the translation (no text)
  splits/images.jsonl    the 4,949 real images: record, split, source URL, localization box, test domain
  preds/*.jsonl          test predictions of the main systems; Gamla runor references removed
  gamla_runor/fetch_gamla_runor.py   retrieves the Gamla runor texts and photographs from christerhamp.se
  code/                  corpus construction, training, inference and evaluation scripts
"""
import hashlib
import json
import os
import shutil
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from runic_norm import normalize  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = f"{ROOT}/release_access"
shutil.rmtree(OUT, ignore_errors=True)
for d in ("splits", "preds", "gamla_runor", "code/jobs"):
    os.makedirs(f"{OUT}/{d}", exist_ok=True)


def sha(s):
    return hashlib.sha1((s or "").encode("utf-8")).hexdigest()


def rd(p):
    return [json.loads(l) for l in open(p) if l.strip()]


corpus = {r["id"]: r for r in rd(f"{ROOT}/data/corpus.jsonl")}
lf = json.load(open(f"{ROOT}/data/leakfree.json"))
keep = {"train": None, "val": set(lf["keep_val"]), "test": set(lf["keep_test"])}

# ---------------- pairs ----------------
n = 0
with open(f"{OUT}/splits/pairs.jsonl", "w") as f:
    for split in ("train", "val", "test"):
        for r in rd(f"{ROOT}/splits_local/mt_{split}.jsonl"):
            if keep[split] is not None and r["id"] not in keep[split]:
                continue
            c = corpus[r["id"]]
            rec = dict(id=r["id"], source={"srd": "rundata", "rdb": "runes", "ch": "gamla_runor"}[c["src"]],
                       signum=c.get("signum"), split=split, cluster=int(c["cluster"]), lang=c["lang"], page=c.get("page"))
            if c["src"] == "ch":
                rec.update(image_urls=c.get("image_urls"), sha1_transliteration=sha(r["src"]), sha1_translation=sha(r["tgt"]))
            else:
                rec.update(transliteration_raw=c["translit_raw"], transliteration=r["src"], translation=r["tgt"])
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
            n += 1
print("pairs", n)

# ---------------- images ----------------
boxes = {r["image"]: r["crop_box"] for r in rd(f"{ROOT}/data/crop_boxes.jsonl")}
manifest = {m["file"]: m for m in rd(f"{ROOT}/data/image_manifest.jsonl")}
card = json.load(open(f"{ROOT}/data/card_labels.json"))["labels"]
cream = json.load(open(f"{ROOT}/data/cream_share.json"))


def domain(r):
    if r["source"] == "ch":
        return "gamla_runor_photo"
    img = r["image"]
    if img in card:
        return "runes_photo" if card[img] in ("photo", "manuscript") else "runes_card"
    return "runes_card" if cream.get(img, 0) > 0.2 else "runes_photo"


n = 0
with open(f"{OUT}/splits/images.jsonl", "w") as f:
    for split in ("train", "val", "test"):
        for r in rd(f"{ROOT}/splits_server/ocr_real_{split}.jsonl"):
            if keep[split] is not None and r["row_id"] not in keep[split]:
                continue
            m = manifest.get(os.path.basename(r["image"]), {})
            rec = dict(id=r["id"], record=r["row_id"], split=split, cluster=r["cluster"], url=m.get("url"),
                       crop_box_0_1000=boxes.get(r["image"]), domain=domain(r))
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")
            n += 1
print("images", n)

# ---------------- predictions (Gamla runor references removed) ----------------
PRED = ["ocr_test_ocr4b_cropcut", "ocr_test_ocr4b_mixcut", "e2e_ocr4b_cropcut_mt4b_fromocr", "e2e_ocr4b_cropcut_mt4b_noise_fromocr",
        "e2e_ocr4b_mixcut_mt4b_fromgold", "mt_test_mt4b", "mt_test_mt4b_noise", "mt_test_mt08", "mt_test_retrieval"]
DROP = ("target", "tgt", "src", "src_clean", "translit")
for p in PRED:
    rows = rd(f"{ROOT}/preds/{p}.jsonl")
    with open(f"{OUT}/preds/{p}.jsonl", "w") as f:
        for r in rows:
            rid = r.get("row_id") or r["id"]
            if rid not in keep["test"]:
                continue
            if rid.startswith("ch"):
                r = {k: v for k, v in r.items() if k not in DROP}
                if p.startswith("mt_test") or p.endswith("fromgold"):
                    r = {k: v for k, v in r.items() if k != "pred"} | {"pred_sha1": sha(r.get("pred"))}
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
print("preds", len(PRED))

# ---------------- code ----------------
CODE = ["runic_norm.py", "build_corpus.py", "relink.py", "make_splits.py", "doc_filter.py", "red_share.py", "synth.py",
        "gen_diffusion.py", "ground.py", "make_crops.py", "qwen_common.py", "train_lora.py", "generate.py", "ocr_noise.py",
        "noise_curve.py", "retrieval_baseline.py", "prior_baselines.py", "loopcut.py", "cutpreds.py", "score_ocr.py",
        "metrics.py", "analyze.py", "make_tables.py", "make_tables_rev.py", "crop_mechanism.py", "matched_noise.py",
        "names_e2e.py", "label_sensitivity.py", "perfilev_infer.py", "valsel.py", "backbones.py", "gold_eval.py",
        "fig_noise.py", "build_release.py"]
for c in CODE:
    if os.path.exists(f"{ROOT}/code/{c}"):
        shutil.copy(f"{ROOT}/code/{c}", f"{OUT}/code/{c}")
    else:
        print("missing code file", c)
for j in ("exp_v2.sh", "exp_v4.sh", "exp_v6.sh", "exp_v8.sh", "exp_v9.sh", "common.sh"):
    shutil.copy(f"{ROOT}/code/jobs/{j}", f"{OUT}/code/jobs/{j}")
shutil.copy(f"{ROOT}/release_access_extra/fetch_gamla_runor.py", f"{OUT}/gamla_runor/fetch_gamla_runor.py")
leak = subprocess.run(["grep", "-rlE", r"hf_[A-Za-z0-9]{30,}|sk-[A-Za-z0-9]{20,}|BEGIN (RSA|OPENSSH) PRIVATE", OUT],
                      capture_output=True, text=True).stdout
print("token scan:", leak or "clean")