"""Build the MiniWeb data release: the paper's data, consolidated, split-labelled and free of personal information.

    python scripts/build_release.py [--out data/release/miniweb-v1] [--no-generated]

Reads the working data (data/annotations, data/datagen, data/webmix, data/sites.yaml, data/macros.yaml,
data/final_set.yaml) and writes one self-contained folder (layout and field definitions in its README.md):

  sites.jsonl                      the 65 sites: domain, split, hub, data source and license
  primitives/                      the macro registry, the final 37-primitive set and the 7 skill families
  tasks/{test,train,excluded}/     the 369 human tasks by split (test: the 62 that start on a held-out site; train: the
    <task_id>/                     304 on training sites only; excluded: 3 that start on a training site and use a held-out
                                   site, in neither split), each with its current (repaired) definition and verifier, the
                                   annotator's original definition and verifier from before the review (original/), the
                                   human demonstration (demonstration/) and, where a reviewer re-walked the task, that walk
  tasks/index.jsonl                one line per task: split, primitives, author, what was repaired, recording provenance and
                                   validity (valid / stale / rejected by the current verifier, as in the paper's Supp.)
  generated/                       MiniWebGen: the 3,128 kept trajectories (training sites only), the 4,614 training
                                   episodes built from them (plain / recovery / chain), the per-primitive guides, and the
                                   generated single-primitive test tasks of the held-out sites

Personal information: annotators appear only as pseudonyms (A1..A6, in order of their first task), derived from the data
at build time so no name is written in this script; their names are replaced in every field and free-text note, and in
site content they typed (as a whole value, so synthetic full names such as a site user's stay); the deployment host the
annotators used is replaced by the local server's address. The build ends with an audit of the written files and fails
if any annotator name or the deployment host remains outside synthetic site content.
"""
from __future__ import annotations

import argparse
import collections
import json
import os
import re
import shutil
import sys
import unicodedata
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
DATA = ROOT / "data"
ANN = DATA / "annotations"
HOST_RE = re.compile(r"https?://[\w.-]+\.up\.railway\.app|[\w.-]+\.up\.railway\.app")
LOCAL = "http://localhost:8080"
PERSON_KEYS = re.compile(r"^(annotator|reviewer|reviewed_by|rerecorded_by|verification_walk_by|walk_by|author|actor|by|"
                         r"recorded_by|edited_by|updated_by|user)$")
# a task's content fields: an original and a current definition differ in what is graded or shown to the agent
CONTENT_KEYS = ("instruction", "sites", "macros", "macro_edges", "macro_subtasks", "macro_spans", "macro_operations",
                "macro_required", "expected_answer", "answer_type", "alternatives", "expected_outcome", "starting_url",
                "requires_login", "qa_answers")
HUMAN_FILES = ("trajectory.json", "server_log.json", "beacon_log.json")


def fold(s):
    return "".join(c for c in unicodedata.normalize("NFKD", s) if not unicodedata.combining(c)).casefold()


# ── personal information ─────────────────────────────────────────────────────

class Pseudonyms:
    """Annotator names (the annotation folders, and every person field of every task) -> A1..An by first task date.
    Spelling variants of one name (case, accents) share a pseudonym."""

    def __init__(self, tasks):
        first, variants = {}, collections.defaultdict(set)
        for folder, t in tasks:
            for v in [folder, t.get("annotator")] + [t.get(k) for k in ("rerecorded_by", "verification_walk_by")]:
                if isinstance(v, str) and v.strip() and not v.lower().startswith(("claude", "ai", "auto")):
                    key = fold(v.strip())
                    variants[key].add(v.strip())
                    first[key] = min(first.get(key, "9"), str(t.get("saved_at") or "9"))
        order = sorted(first, key=lambda k: (first[k], k))
        self.of = {k: f"A{i + 1}" for i, k in enumerate(order)}
        names = sorted({n for vs in variants.values() for n in vs} | {k for k in self.of}, key=len, reverse=True)
        self.name_re = re.compile(r"(?<![\w])(" + "|".join(re.escape(n) for n in names) + r")(?![\w])", re.I)

    def person(self, v):
        if not isinstance(v, str):
            return v
        if v.lower().startswith(("claude", "ai")) or v.lower().endswith("-review"):
            return "AI review"                              # an LLM reviewer (claude-review, codex-review), not a person
        return self.of.get(fold(v.strip()), v)

    @staticmethod
    def in_full_name(s, start, end):
        """Part of a longer name ('Leila Haddad'): a synthetic site user, not an annotator."""
        return bool(re.search(r"[A-Z][a-z]+ $", s[max(0, start - 30):start]) or re.match(r" [A-Z][a-z]", s[end:end + 3]))

    def text(self, s):
        """Names -> pseudonyms in free text and site content, except as part of a longer name: instructions and pages
        mention synthetic users who share a first name with an annotator."""
        return self.name_re.sub(lambda m: m.group(0) if self.in_full_name(m.string, m.start(), m.end())
                                else self.of.get(fold(m.group(0)), m.group(0)), s)

    def scrub(self, obj):
        if isinstance(obj, dict):
            return {k: (self.person(v) if PERSON_KEYS.match(k) and isinstance(v, str) else self.scrub(v)) for k, v in obj.items()}
        if isinstance(obj, list):
            return [self.scrub(v) for v in obj]
        if isinstance(obj, str):
            return HOST_RE.sub(lambda m: LOCAL if m.group(0).startswith("http") else "localhost:8080", self.text(obj))
        return obj


def write_json(path, obj):
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(obj, indent=1, ensure_ascii=False))


def link_tree(src, dst):
    """Images: hard links (same disk, no copy), else copies."""
    for p in Path(src).rglob("*"):
        if p.is_file():
            q = dst / p.relative_to(src)
            q.parent.mkdir(parents=True, exist_ok=True)
            if not q.exists():
                try:
                    os.link(p, q)
                except OSError:
                    shutil.copy2(p, q)


# ── human tasks ──────────────────────────────────────────────────────────────

def recording_status(tasks):
    """valid / stale / verifier_rejected per task, as the paper's grader validation decides it (Supp. graders):
    stale if tagged stale, else whether the task's current verifier passes the gold recording."""
    from miniweb.annotation.app import _load_test_trajectory
    from miniwebagent.evaluate import grade_human, human_tasks
    by_id = {t["task_id"]: t for t in human_tasks(split="all")}
    out = {}
    for folder, t in tasks:
        tid = t["task_id"]
        if ((t.get("review_tag") or {}).get("tag")) == "stale":
            out[tid] = "stale"
            continue
        traj, answer, _ = _load_test_trajectory(folder, tid, "gold")
        out[tid] = "valid" if grade_human(by_id[tid], traj or [], answer)[0] else "verifier_rejected"
    return out


def splits():
    from miniwebagent.evaluate import human_tasks
    test = {t["task_id"] for t in human_tasks(split="test")}
    train = {t["task_id"] for t in human_tasks(split="train")}
    return test, train


def build_tasks(out, P, tasks, status):
    test, train = splits()
    index = []
    for folder, t in tasks:
        tid = t["task_id"]
        src = ANN / folder / Path(t["_dir"]).name
        split = "test" if tid in test else "train" if tid in train else "excluded"
        d = out / "tasks" / split / tid
        cur_v = json.loads((src / "verifier.json").read_text())
        before = json.loads((src / "before_review.json").read_text()) if (src / "before_review.json").exists() else {}
        orig_t, orig_v = before.get("task") or {}, before.get("verifier") or {}
        task = {k: v for k, v in t.items() if k != "_dir"}
        write_json(d / "task.json", P.scrub(task))
        write_json(d / "verifier.json", P.scrub(cur_v))
        if orig_t:
            write_json(d / "original" / "task.json", P.scrub(orig_t))
            write_json(d / "original" / "verifier.json", P.scrub(orig_v))
        for f in HUMAN_FILES:
            if (src / f).exists():
                write_json(d / "demonstration" / f, P.scrub(json.loads((src / f).read_text())))
        if (src / "screenshots").is_dir():
            link_tree(src / "screenshots", d / "demonstration" / "screenshots")
        if (src / "verification_walk.json").exists():
            write_json(d / "verification_walk" / "walk.json",
                       P.scrub(json.loads((src / "verification_walk.json").read_text())))
        changed = [k for k in CONTENT_KEYS if orig_t and k in orig_t and orig_t.get(k) != t.get(k)]
        if t.get("instruction_original") and t.get("instruction_original") != t.get("instruction") and "instruction" not in changed:
            changed.append("instruction")
        review = t.get("ai_review") or {}
        prov = {"task_id": tid, "split": split, "author": P.person(t.get("annotator")),
                "task": {"repaired": bool(changed), "changed_fields": changed,
                         "original": "original/task.json" if orig_t else None,
                         "review": {"recommendation": review.get("recommendation"),
                                    "changes": P.scrub(review.get("changes") or [])}},
                "verifier": {"repaired": bool(orig_v) and orig_v.get("macros") != cur_v.get("macros"),
                             "original": "original/verifier.json" if orig_v else None},
                "demonstration": {"provenance": "re-recorded" if t.get("rerecorded_at") else "original",
                                  "recorded_by": P.person(t.get("rerecorded_by") or t.get("annotator")),
                                  "rerecorded_at": t.get("rerecorded_at"), "status": status[tid],
                                  "screenshots": (src / "screenshots").is_dir()},
                "verification_walk": {"present": (src / "verification_walk.json").exists(),
                                      "walked_by": P.scrub(t.get("verification_walk_by"))}}
        write_json(d / "provenance.json", prov)
        index.append({"task_id": tid, "split": split, "site": t.get("site"),
                      "sites": [s["id"] if isinstance(s, dict) else s for s in (t.get("sites") or [t.get("site")])],
                      "primitives": [m for m in (t.get("macros") or []) if m], "author": prov["author"],
                      "task_repaired": prov["task"]["repaired"], "changed_fields": changed,
                      "verifier_repaired": prov["verifier"]["repaired"],
                      "demonstration": prov["demonstration"]["provenance"], "demonstration_status": status[tid],
                      "verification_walk": prov["verification_walk"]["present"], "path": f"tasks/{split}/{tid}"})
    (out / "tasks" / "index.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in index))
    return index


# ── generated data (MiniWebGen) ──────────────────────────────────────────────

def build_generated(out, train_sites):
    pool = (ROOT / "scripts" / "pools" / "pool3k_plain_ids.txt").read_text().split()
    episodes = (ROOT / "scripts" / "pools" / "pool3k_dirs.txt").read_text().split()
    kept = {}
    for k in sorted((DATA / "datagen" / "runs").glob("*/kept.jsonl")):
        for line in k.read_text().split("\n"):
            if line.strip():
                r = json.loads(line)
                kept[r["task_id"]] = {**r, "run": k.parent.name}
    g = out / "generated"
    tindex = []
    missing = []
    for tid in pool:
        r = kept[tid]
        assert r["site"] in train_sites, (tid, r["site"])
        acc = DATA / "datagen" / "runs" / r["run"] / "accepted" / tid
        if not (acc / "trajectory.json").exists():
            missing.append(tid)
            continue
        d = g / "trajectories" / tid
        for f in ("trajectory.json", "elements.json"):
            if (acc / f).exists():
                write_json(d / f, json.loads((acc / f).read_text()))
        if (acc / "screenshots").is_dir():
            link_tree(acc / "screenshots", d / "screenshots")
        meta = {k: r.get(k) for k in ("task_id", "macro", "site", "start_kind", "instruction", "backend", "judge", "run")}
        write_json(d / "meta.json", meta)
        tindex.append({k: meta[k] for k in ("task_id", "macro", "site", "start_kind")} | {"path": f"generated/trajectories/{tid}"})
    eindex, variants = [], collections.Counter()
    for e in episodes:
        src = DATA / "webmix" / "replay_v4" / e
        ep = json.loads((src / "episode.json").read_text())
        ep.pop("response_format", None)
        variant = ep.get("variant") or "plain"
        variants[variant] += 1
        d = g / "training_episodes" / e
        write_json(d / "episode.json", {**ep, "variant": variant})
        shutil.copy2(src / "rows.jsonl", d / "rows.jsonl")
        for p in src.glob("step_*.png"):
            if not (d / p.name).exists():
                try:
                    os.link(p, d / p.name)
                except OSError:
                    shutil.copy2(p, d / p.name)
        steps = sum(1 for line in (src / "rows.jsonl").read_text().split("\n") if line.strip())
        eindex.append({"episode": e, "variant": variant, "site": ep.get("site"), "macro": ep.get("macro"),
                       "parts": ep.get("parts") or [e.split("~")[0]], "steps": steps,
                       "path": f"generated/training_episodes/{e}"})
    (g / "trajectories" / "index.jsonl").write_text("".join(json.dumps(r) + "\n" for r in tindex))
    (g / "training_episodes" / "index.jsonl").write_text("".join(json.dumps(r) + "\n" for r in eindex))
    gd = g / "guides"
    gd.mkdir(parents=True, exist_ok=True)
    for p in sorted((DATA / "datagen" / "guides").glob("*.md")):
        shutil.copy2(p, gd / p.name)
    shutil.copy2(DATA / "webmix" / "valgen" / "val_v2.jsonl", g / "single_primitive_test_tasks.jsonl")
    return {"trajectories": len(tindex), "missing_exports": missing, "episodes": len(eindex), "variants": dict(variants),
            "steps": sum(r["steps"] for r in eindex), "guides": len(list(gd.glob("*.md")))}


def build_task_centric(out, train_sites):
    """The task-centric comparison data of Tab. 3 (miniwebgen.taskgen, InSTA's pipeline on the training sites): every
    proposed task with its attempt's judged outcome, and the selected episodes in the agent harness's row format with
    the judge's verdict."""
    T = DATA / "webmix" / "taskgen"
    tasks = [json.loads(line) for line in (T / "runs" / "v1" / "tasks.jsonl").read_text().split("\n") if line.strip()]
    judged = {f.stem: json.loads(f.read_text()) for f in (T / "runs" / "v1_judge").glob("tg-*.json")}
    selected = sorted(p.name for p in (T / "rows_27k").iterdir() if p.is_dir() and p.name.startswith("tg-"))
    g = out / "generated" / "task_centric"
    rows = 0
    for e in selected:
        src, d = T / "rows_27k" / e, g / "episodes" / e
        write_json(d / "episode.json", json.loads((src / "episode.json").read_text()))
        shutil.copy2(src / "rows.jsonl", d / "rows.jsonl")
        rows += sum(1 for line in (src / "rows.jsonl").read_text().split("\n") if line.strip())
        for img in src.glob("step_*.png"):
            if not (d / img.name).exists():
                try:
                    os.link(img, d / img.name)
                except OSError:
                    shutil.copy2(img, d / img.name)
        if e in judged:
            write_json(d / "judge.json", judged[e])
    chosen = set(selected)
    index = []
    for t in tasks:
        assert t["site"] in train_sites, (t["task_id"], t["site"])
        j = judged.get(t["task_id"])
        index.append({k: t.get(k) for k in ("task_id", "site", "instruction", "start_path", "task_kind")}
                     | {"judged_success": None if j is None else bool(j.get("predicted_label")),
                        "selected": t["task_id"] in chosen})
    (g / "tasks.jsonl").write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in index))
    return {"proposed": len(tasks), "judged": len(judged),
            "judged_success": sum(bool(j.get("predicted_label")) for j in judged.values()),
            "episodes": len(selected), "rows": rows, "sites": len({t["site"] for t in index if t["selected"]})}


def finalize_generated(out):
    """Per training episode: its rows, the rows that are training targets (an injected mistake and the replayed prefix
    of a recovery episode are not), and whether it falls in the 5% held out for validation loss (webmix.train's rules),
    so the release reproduces the paper's 27.6K fitted steps."""
    from miniwebagent import train as T
    d = out / "generated" / "training_episodes"
    idx = [json.loads(line) for line in (d / "index.jsonl").read_text().split("\n") if line.strip()]
    for r in idx:
        ep = json.loads((d / r["episode"] / "episode.json").read_text())
        rows = [json.loads(line) for line in (d / r["episode"] / "rows.jsonl").read_text().split("\n") if line.strip()]
        tgt = [x for x in rows if not (x.get("inject") or x.get("aug_prefix"))]
        for x in tgt:
            x["parts"] = ep.get("parts")
        r.update(rows=len(rows), target_rows=len(tgt), validation=bool(tgt) and T.row_is_val(tgt[0]))
    (d / "index.jsonl").write_text("".join(json.dumps(r) + "\n" for r in idx))
    return {"rows": sum(r["rows"] for r in idx), "target_rows": sum(r["target_rows"] for r in idx),
            "fitted_rows": sum(r["target_rows"] for r in idx if not r["validation"])}


# ── audit ────────────────────────────────────────────────────────────────────

HUMAN_PARTS = ("tasks", "generated/guides")      # what annotators wrote, recorded or reviewed (the guides are distilled
                                                 # from their demonstrations); the rest is generated by code on synthetic sites


def audit(out, P):
    """The deployment host must be gone from every written text file, and annotator names from everything humans touched
    (HUMAN_PARTS and the top-level files). -> (findings, full-name contexts left in place for review: synthetic site users
    who share a first name with an annotator)."""
    bad, kept = collections.Counter(), collections.Counter()
    for p in out.rglob("*"):
        if p.suffix not in (".json", ".jsonl", ".md", ".yaml") or not p.is_file():
            continue
        s = p.read_text(errors="ignore")
        if HOST_RE.search(s):
            bad[("host", str(p.relative_to(out)))] += 1
        rel = str(p.relative_to(out))
        if "/" in rel and not rel.startswith(HUMAN_PARTS):
            continue
        for m in P.name_re.finditer(s):
            ctx = re.sub(r"\s+", " ", s[max(0, m.start() - 25):m.end() + 15])
            if P.in_full_name(s, m.start(), m.end()):
                kept[re.sub(r"\s+", " ", s[max(0, m.start() - 12):m.end() + 10]).strip()] += 1
            else:
                bad[("name", str(p.relative_to(out)), ctx)] += 1
    return bad, kept


# ── data card ─────────────────────────────────────────────────────────────────

def write_card(out, sm):
    """README.md of the release, every number from this build's summary."""
    t, d, g, rp = sm["tasks"], sm["demonstrations"], sm.get("generated") or {}, sm.get("repaired_parts", {})
    n = sum(t.values())
    v, tc = g.get("variants", {}), g.get("task_centric") or {}
    text = f"""# MiniWeb data, release v1

The data behind the MiniWeb paper: {sm['sites']} synthetic websites, {n} human-written tasks with their verifiers and
demonstrations, and the MiniWebGen training data generated on the training sites. Built by `scripts/build_release.py`
from the working data; every count below comes from `summary.json`.

## Layout

```
sites.jsonl                  {sm['sites']} sites: domain, split, hub, data source, license
primitives/                  macros.yaml (46 base macros, 7 reasoning operations), final_set.yaml (the 37 primitives
                             of the final set and the 9 dropped), families.json (7 skill families)
tasks/index.jsonl            one line per human task: split, primitives, author, what was repaired, demonstration
tasks/<split>/<task_id>/
  task.json                  the task as evaluated (after review)
  verifier.json              its verifier as evaluated
  original/task.json         the annotator's task before the review
  original/verifier.json     the annotator's verifier before the review
  demonstration/             the human recording: trajectory.json (actions and observations), server_log.json
                             (backend requests), beacon_log.json (input events), screenshots/
  verification_walk/walk.json   a second walk-through by a reviewer, where one exists
  provenance.json            split, author, repaired parts, demonstration provenance and status
generated/
  trajectories/<task_id>/    MiniWebGen's kept single-primitive trajectories: trajectory.json, elements.json,
                             screenshots/, meta.json (primitive, site, instruction, backend verdict)
  training_episodes/<id>/    the training episodes built from them: episode.json, rows.jsonl (prompt and target per
                             step, in the agent harness's format), step_*.png
  guides/                    the per-primitive guides MiniWebGen distills from human demonstrations
  single_primitive_test_tasks.jsonl   generated single-primitive tasks of the held-out sites (evaluation only)
  task_centric/              the task-centric comparison data: tasks.jsonl (every proposed task, its judged outcome,
                             whether it was selected), episodes/<task_id>/ (episode.json, rows.jsonl, step_*.png,
                             judge.json)
```

## Splits

The 13 held-out (test) sites were fixed before any data was generated; the other 52 are training sites
(`sites.jsonl`, field `split`).

- `tasks/test/`: {t.get('test', 0)} tasks that start on a held-out site. The evaluation set of the paper.
- `tasks/train/`: {t.get('train', 0)} tasks that use training sites only. Used for development and as human
  demonstrations for MiniWebGen's guides.
- `tasks/excluded/`: {t.get('excluded', 0)} tasks that start on a training site and also use a held-out site. Neither
  split uses them.
- `generated/` holds training-site data only. `single_primitive_test_tasks.jsonl` is the exception: tasks generated on
  the held-out sites for evaluation, never executed for training.

## Original and repaired

Every human task went through a review after annotation, and the release keeps both versions.

- `original/` is the annotator's task and verifier before the review; `task.json` and `verifier.json` are the
  versions the paper evaluates. `provenance.json` lists the fields that changed. Across the {n} tasks the review
  reworded {rp.get('instruction', 0)} instructions, changed the primitive chain of {rp.get('macros', 0)}, re-marked
  primitive spans in {rp.get('macro_spans', 0)}, changed the expected answer of {rp.get('expected_answer', 0)} and the
  start page of {rp.get('starting_url', 0)}, and changed {sm.get('verifier_repaired', 0)} verifiers.
- `demonstration/` is the annotator's original recording for {n - sm.get('re_recorded', 0)} tasks and a re-recording
  for {sm.get('re_recorded', 0)} (field `demonstration.provenance` is `original` or `re-recorded`; the replaced
  recordings were not kept). {sm.get('verification_walks', 0)} tasks also have a reviewer's walk-through.
- `demonstration.status` says whether the recording still stands in for a correct episode, as the paper's grader
  validation decides it: `valid` ({d.get('valid', 0)}) passes the task's current verifier; `stale` ({d.get('stale', 0)})
  predates a change to its site, data or task; `verifier_rejected` ({d.get('verifier_rejected', 0)}) fails the current
  verifier, for example an early recording whose recorder missed a form submission. Fuzzy text checks are judged by
  an LLM, so a borderline recording can pass in one run and fail in another.

## Generated data (MiniWebGen)

- {g.get('trajectories', 0)} kept trajectories, each verified by the backend change it was meant to cause.
- {g.get('episodes', 0)} training episodes (`variant` in `episode.json` and the index): {v.get('plain', 0)} `plain`
  replays of those trajectories, {v.get('recovery', 0)} `recovery` episodes that inject a mistake and then correct it,
  and {v.get('chain', 0)} `chain` episodes that join two trajectories of one site.
- {g.get('rows', 0):,} rows. In a recovery episode the injected mistake and the replayed prefix before it are marked
  (`inject`, `aug_prefix`) and are not training targets, which leaves {g.get('target_rows', 0):,}; holding out 5% of the
  trajectories for validation loss (`validation` in the index) leaves the {g.get('fitted_rows', 0):,} steps the paper's
  executor is fitted on.

## Task-centric comparison data

The paper compares MiniWebGen's skill-centric data with task-centric data of the same size, collected on the same
training sites with a reimplementation of InSTA's pipeline (`src/miniwebgen/taskgen.py`). Gemini 3.5 Flash proposed
{tc.get('proposed', 0):,} goal-level tasks; the untrained Qwen3.5-4B attempted each in the agent harness, writing every step;
Gemini 3.5 Flash judged {tc.get('judged', 0):,} episodes from their screenshots, actions, the server's request log and
data changes, and accepted {tc.get('judged_success', 0):,}. {tc.get('episodes', 0):,} of them, drawn round-robin over
{tc.get('sites', 0)} sites, make the {tc.get('rows', 0):,} training rows. The judge is not a verifier: on the human tasks of
the training sites it agrees with the task verifiers on 87.1% of episodes, with 77.5% precision, so some selected
episodes do not complete their task. `judge.json` keeps its verdict and its reasons per part of the task.

## Personal information

- Annotators appear only as pseudonyms A1 to A{sm.get('authors', 6)}, in the order of their first task, in every field,
  review note and walk-through. AI reviewers appear as `AI review`.
- Addresses of the deployment the annotators worked on are replaced with the local server's address.
- The build audits every file it writes and fails if an annotator name or that address remains. Names of site users
  (for example `Leila Haddad`) are synthetic and are kept.
- Sites seeded from public datasets show those datasets' content, including names and addresses in the Enron email
  corpus and user names from the WebArena Reddit and GitLab environments, under the licenses in `sites.jsonl`.
"""
    (out / "README.md").write_text(text)


def main(argv=None):
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=str(DATA / "release" / "miniweb-v1"))
    ap.add_argument("--no-generated", action="store_true")
    ap.add_argument("--audit-only", action="store_true", help="re-run the audit on an existing build (updates summary.json)")
    ap.add_argument("--finalize", action="store_true", help="training-row counts and the data card for an existing build")
    a = ap.parse_args(argv)
    out = Path(a.out)
    if a.finalize:
        summary = json.loads((out / "summary.json").read_text())
        summary["generated"].update(finalize_generated(out))
        summary["generated"].pop("steps", None)
        write_json(out / "summary.json", summary)
        write_card(out, summary)
        print(json.dumps(summary["generated"], indent=1))
        return
    if a.audit_only:
        tasks = [(f.parent.parent.name, json.loads(f.read_text())) for f in sorted(ANN.glob("*/*/task.json"))]
        P = Pseudonyms(tasks)
        bad, kept = audit(out, P)
        summary = json.loads((out / "summary.json").read_text())
        summary.update(audit_findings=len(bad))
        summary.pop("full_names_kept", None)
        write_json(out / "summary.json", summary)
        for k, n in list(bad.items())[:20]:
            print("  AUDIT", n, k)
        print(f"audit: {len(bad)} findings; {sum(kept.values())} synthetic full names kept")
        sys.exit("personal information left in the release" if bad else 0)
    if out.exists():
        shutil.rmtree(out)                                   # a build output, rebuilt from data/ every time
    out.mkdir(parents=True)
    tasks = []
    for f in sorted(ANN.glob("*/*/task.json")):
        t = json.loads(f.read_text())
        t["_dir"] = str(f.parent)
        tasks.append((f.parent.parent.name, t))
    P = Pseudonyms(tasks)
    import yaml
    sites = yaml.safe_load((DATA / "sites.yaml").read_text())["sites"]
    (out / "sites.jsonl").write_text("".join(json.dumps(s, ensure_ascii=False) + "\n" for s in sites))
    prim = out / "primitives"
    prim.mkdir()
    for f in ("macros.yaml", "final_set.yaml"):
        shutil.copy2(DATA / f, prim / f)
    from miniwebgen.viewer import FAMILIES
    write_json(prim / "families.json", FAMILIES)
    print(f"grading the {len(tasks)} demonstrations with their current verifiers ...", flush=True)
    status = recording_status(tasks)
    index = build_tasks(out, P, tasks, status)
    gen = None if a.no_generated else build_generated(out, {s["id"] for s in sites if s["split"] == "train"})
    if gen:
        gen.pop("steps", None)
        gen.update(finalize_generated(out))
        gen["task_centric"] = build_task_centric(out, {s["id"] for s in sites if s["split"] == "train"})
    bad, kept = audit(out, P)
    summary = {"sites": len(sites), "tasks": collections.Counter(r["split"] for r in index),
               "demonstrations": collections.Counter(r["demonstration_status"] for r in index),
               "re_recorded": sum(r["demonstration"] == "re-recorded" for r in index),
               "task_repaired": sum(r["task_repaired"] for r in index),
               "repaired_parts": collections.Counter(f for r in index for f in r["changed_fields"]),
               "verifier_repaired": sum(r["verifier_repaired"] for r in index),
               "verification_walks": sum(r["verification_walk"] for r in index), "authors": len(P.of),
               "generated": gen, "audit_findings": len(bad)}
    # the kept full names go to the log only: listed in the release they would show which first names annotators have
    print("synthetic full names kept:", dict(kept.most_common(30)), flush=True)
    write_json(out / "summary.json", summary)
    write_card(out, json.loads((out / "summary.json").read_text()))
    print(json.dumps(summary, indent=1, default=dict))
    for k, n in list(bad.items())[:20]:
        print("  AUDIT", n, k)
    if bad:
        sys.exit("personal information left in the release (see above)")


if __name__ == "__main__":
    main()