Blame for scripts/demo_setup.py

"""Prepare the public demo (MINIWEB_DEMO=1, src/miniweb/app/demo.py) from the Hugging Face release. The Space runs it on
every start, before the server (deploy/space/start.sh); everything goes under one folder, data/demo/ by default.

    python scripts/demo_setup.py                              # downloads from the Hub
    python scripts/demo_setup.py --no-media                   # without the 8 GB of site images
    python scripts/demo_setup.py --src data/release/miniweb-v1-hf   # from a local copy of the release

Each step is skipped when its output exists:
1. the site database, static images and media (scripts/restore_site_db.py): miniweb.db, static/, media/
2. the human tasks with their recordings (scripts/fetch_tasks.py): annotations/, which the annotation tool shows
3. translation_replay.json: each translation the recordings requested, with the translation the site returned, so
   the demo's translation site answers those requests exactly as recorded
4. the session tables, so the server starts on a ready database
Then the task folders are made read-only. Serve with the variables this prints (start.sh sets them).
"""
import argparse, glob, json, os, stat, subprocess, sys

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
TRANSLATE = "/sites/translation/api/translate"


def run(*args):
    print("+", " ".join(args), flush=True)
    subprocess.run([sys.executable, *args], check=True, cwd=ROOT)


def recorded_translations(annotations):
    """{(source, target, text): translation} from the tasks' server logs and verification walks."""
    out, conflicts = {}, 0

    def walk(o):
        nonlocal conflicts
        if isinstance(o, dict):
            body, resp = o.get("body"), o.get("response")
            if (o.get("path") == TRANSLATE and isinstance(body, dict) and isinstance(resp, dict)
                    and body.get("text") and resp.get("translated_text")):
                key = (body.get("source", "").strip(), body.get("target", "").strip(), body["text"].strip())
                if out.setdefault(key, resp["translated_text"]) != resp["translated_text"]:
                    conflicts += 1                  # two recordings disagree: the first one read stays
            for v in o.values():
                walk(v)
        elif isinstance(o, list):
            for v in o:
                walk(v)

    for f in sorted(glob.glob(os.path.join(annotations, "*", "*", "server_log.json"))
                    + glob.glob(os.path.join(annotations, "*", "*", "verification_walk.json"))):
        with open(f) as fh:
            walk(json.load(fh))
    if conflicts:
        print(f"warning: {conflicts} recorded translations disagree with an earlier recording of the same text")
    return out


def read_only(folder):
    for d, _, files in os.walk(folder):
        for f in files:
            os.chmod(os.path.join(d, f), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
        os.chmod(d, stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--src", help="a local copy of the release (default: download from the Hub)")
    ap.add_argument("--out", default=os.path.join(ROOT, "data", "demo"))
    ap.add_argument("--no-media", action="store_true")
    a = ap.parse_args()
    out = os.path.abspath(a.out)
    paths = {"MINIWEB_DB": os.path.join(out, "miniweb.db"), "MINIWEB_DATA_STATIC": os.path.join(out, "static"),
             "MINIWEB_MEDIA_DIR": os.path.join(out, "media"), "MINIWEB_ANNOTATIONS_DIR": os.path.join(out, "annotations"),
             "MINIWEB_TRANSLATION_REPLAY": os.path.join(out, "translation_replay.json")}
    os.makedirs(out, exist_ok=True)

    if not os.path.exists(paths["MINIWEB_DB"]):
        run("scripts/restore_site_db.py", "--db", paths["MINIWEB_DB"], "--static", paths["MINIWEB_DATA_STATIC"],
            "--media", paths["MINIWEB_MEDIA_DIR"], *(["--src", os.path.join(a.src, "site_db")] if a.src else []),
            *(["--no-media"] if a.no_media else []))
    if not os.path.isdir(paths["MINIWEB_ANNOTATIONS_DIR"]):
        run("scripts/fetch_tasks.py", "--out", paths["MINIWEB_ANNOTATIONS_DIR"], "--valset", os.devnull,
            *(["--src", a.src] if a.src else []))
    if not os.path.exists(paths["MINIWEB_TRANSLATION_REPLAY"]):
        rows = [{"source": s, "target": t, "text": x, "translated": y}
                for (s, t, x), y in sorted(recorded_translations(paths["MINIWEB_ANNOTATIONS_DIR"]).items())]
        with open(paths["MINIWEB_TRANSLATION_REPLAY"], "w") as fh:
            json.dump(rows, fh, indent=1, ensure_ascii=False)
        print(f"{len(rows)} recorded translations -> {paths['MINIWEB_TRANSLATION_REPLAY']}")

    os.environ["MINIWEB_DB"] = paths["MINIWEB_DB"]
    sys.path.insert(0, os.path.join(ROOT, "src"))
    from miniweb.app import db
    db.init_db()
    db.close()
    read_only(paths["MINIWEB_ANNOTATIONS_DIR"])

    print("serve with:\n" + "\n".join(f"  {k}={v}" for k, v in paths.items()) + "\n  MINIWEB_DEMO=1")


if __name__ == "__main__":
    main()