#!/usr/bin/env python3 """Build the composed release: main's code and corpus with the APPROVED curated JSON. python tools/compose_release.py [--code-ref main] [--approved COMMIT] [--workdir DIR] [--branch release-composed] [--no-embed] [--no-commit] [--keep] [--allow-test-failures] Held-for-review curated content and main's code share one tree, so a code fix cannot ship on its own (roadmap T0.1). This builds the tree that can, WITHOUT touching the main working tree -- everything happens in a detached `git worktree`: 1. `git worktree add --detach `: main's code, tests and data/processed (the official corpus -- legislation, guidance, case law). 2. data/curated/*.json <- the approved commit's files (data/curated/review_state.json). A curated JSON file the approved commit lacks is held new content and is left out; metadata files (review_state.json) stay. data/curated/held_manifest.json is then written: the ids and MATCHING WORDS of the disposition entries, equivalency pairings and statute-code readings on the code commit that the approved files lack (held for review), so the tools answer a record that reaches only held content 'HELD FOR REVIEW -- not in this deployment' instead of a bare no-match. --review-state PATH composes against another review state (a trial baseline; needs --no-commit). 3. data/processed/commentary.json re-rendered from those files by the repo's own renderer (`python -m canlex.commentary --allow-shrink` inside the worktree -- the approved files render fewer chunks than main's, which the shrink guard would refuse). 4. If commentary.json changed: `python -m canlex.embed` (incremental -- embeddings.npz keys each chunk's vectors by a content hash of its embed text, so only commentary chunks whose text changed are encoded and every other row is reused). --no-embed skips it; the release is then NOT committed, because the index would zero-fill the changed chunks (BM25 only) -- with --no-commit the stale count is printed as a warning and the dry run exits 0 when the composed tree's tests pass. Runtime is dominated by loading and hashing the ~190 MB corpus and rewriting the ~55 MB npz, plus a CPU encode of the changed commentary chunks (on the order of a hundred): minutes, not the ~40 of a --full pass. Not yet timed on this machine -- record it in RUNBOOK.md on the first real run. 5. The offline suite inside the worktree. Tests that assert held content skip there; a failure blocks the commit (--allow-test-failures overrides it -- never for a deploy). 6. data/curated/drafts <- the approved commit's subtree (review memos and the held batches' source drafts ship with their batch; the review check holds any difference), then commit on the worktree's detached HEAD and run the worktree's own tools/predeploy.py on the commit (blobs, commentary-from-curated with the tree's authority registry, embeddings coverage, review, review-code), then point `--branch` at it and print the commit and tree for the RUNBOOK commit-tree recipe. 7. `git worktree remove` (unless --keep). Exit status 0 only when a release commit passed every check (or, with --no-commit, when the composed tree passed its tests). """ import argparse import contextlib import json import os import shutil import subprocess import sys import tempfile import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import predeploy as pd # noqa: E402 (shared git / check helpers) ROOT = pd.ROOT def say(msg): print(msg) sys.stdout.flush() def run(cmd, cwd, env=None, check=True): say(f"$ {' '.join(str(c) for c in cmd)}") t0 = time.monotonic() out = subprocess.run([str(c) for c in cmd], cwd=str(cwd), capture_output=True, env=env) text = (out.stdout + out.stderr).decode("utf-8", "replace") if check and out.returncode != 0: raise RuntimeError(f"{cmd[0]} exited {out.returncode}:\n{text[-4000:]}") return out.returncode, text, time.monotonic() - t0 def remove_worktree(path): if not path.exists(): pd.git("worktree", "prune", check=False) return pd.git("worktree", "remove", "--force", str(path), check=False) pd.git("worktree", "prune", check=False) if path.exists(): shutil.rmtree(path, ignore_errors=True) if path.exists(): raise RuntimeError(f"{path} still exists; remove it by hand") def compose_curated(wt, code, approved, state): """Put the approved curated JSON in the worktree. Returns (replaced, held_removed, restored) file-name lists.""" meta = set(state.get("metadata_files", [])) code_names = pd.curated_names(code) appr_names = pd.curated_names(approved) replaced, held, restored = [], [], [] for name in code_names: target = wt / pd.CURATED_DIR / name # Metadata (review_state.json) always ships from MAIN: the approved # commit's copy predates its own approval, so shipping it would point # the composed tree at the PREVIOUS baseline and fail the baseline # tests. Until 2026-09-15 the approved commit predated the file # entirely, so only the held-removal branch ever needed the guard. if name in meta: continue if name in appr_names: blob = pd.show_bytes(approved, f"{pd.CURATED_DIR}/{name}") if json.loads(blob.decode("utf-8")) != json.loads(target.read_bytes().decode("utf-8")): target.write_bytes(blob) replaced.append(name) else: target.unlink() held.append(name) for name in appr_names: if name not in code_names and name not in meta: (wt / pd.CURATED_DIR / name).write_bytes( pd.show_bytes(approved, f"{pd.CURATED_DIR}/{name}")) restored.append(name) return replaced, held, restored MANIFEST_ABOUT = ( "Written by tools/compose_release.py into a composed release only (main has no such " "file): the ids and MATCHING WORDS of the curated disposition entries and " "equivalency pairings on the code commit that the approved baseline lacks " "(held 'entry' / 'pairing'), the words added to approved ones (held 'names' / " "'terms'), the words the code commit moved from an approved entry to another entry (held " "'moved', with 'moved_from'), the acquittal-reading caveats, routed state rows and readings by " "mechanism the code commit carries on an approved entry (held 'caveat' / 'row' / 'mechanism', " "the last with its 'words'), and the statute-code readings " "the baseline lacks -- all held for review. " "canlex/server.py reads it so a record whose words reach only held content is " "answered 'HELD FOR REVIEW -- not in this deployment' instead of a bare no-match. " "It carries no held legal characterisation.") def _added(new, old, key): have = set((old or {}).get(key) or []) return [t for t in (new or {}).get(key) or [] if t not in have] def _moved(code_disp, appr_disp): """The names and aliases an approved entry carries that the code commit moved to ANOTHER entry: [{id: the entry that has them now, held: 'moved', names, aliases, moved_from: the approved entry}] (the review of batch 1, round 3, RP-04: 'not guilty' moved from dismissed-nolle to the held acquittal entry -- a composed release answers those words from the approved entry, and says the move is held).""" new = {e.get("id"): e for e in code_disp.get("dispositions") or [] if isinstance(e, dict)} out = [] for old in appr_disp.get("dispositions") or []: cur = new.get(old.get("id")) if not isinstance(old, dict) or cur is None: continue for key in ("names", "aliases"): gone = [n for n in old.get(key) or [] if n not in (cur.get(key) or [])] for n in gone: to = next((e["id"] for e in new.values() if e.get("id") != old["id"] and n in (e.get("names") or []) + (e.get("aliases") or [])), None) if to is None: continue item = next((x for x in out if x["id"] == to and x["moved_from"] == old["id"]), None) if item is None: item = {"id": to, "held": "moved", "names": [], "aliases": [], "moved_from": old["id"]} out.append(item) item[key].append(n) return out def _added_gated(e, old): """The gated names (with their places) an entry carries that the approved entry does not (the review of batch 1, round 2: a name added only as a gated name -- 'not proven' -- is held content the manifest must list).""" was = {(n, tuple(g.get("places") or ())) for g in (old or {}).get("gated_names") or [] if isinstance(g, dict) for n in g.get("names") or []} out = [] for g in e.get("gated_names") or []: if not isinstance(g, dict): continue places = list(g.get("places") or []) names = [n for n in g.get("names") or [] if (n, tuple(places)) not in was] if names: out.append({"names": names, "places": places}) return out def _held_caveats(e, old): """The ids of the policy caveats on an approved entry's code-commit version that put the act branch NOT SETTLED (canlex.commentary.caveat_class 'act-not-settled' or 'act-if-acquitted') and that the approved entry lacks (the review of batch 1, round 4, RP-04: an approved ngri, conviction-reversed-on-appeal, mexico-dismissal or pending-charge entry answers on the composed release without the acquittal readings -- the manifest lists the held caveat so the server drafts nothing).""" sys.path.insert(0, str(ROOT)) from canlex.commentary import caveat_class have = {c.get("id") for c in (old or {}).get("policy_caveats") or [] if isinstance(c, dict)} return [c["id"] for c in e.get("policy_caveats") or [] if isinstance(c, dict) and c.get("id") and c["id"] not in have and caveat_class(c) in ("act-not-settled", "act-if-acquitted")] def _held_rows(e, old): """The states whose row on an approved entry's code-commit version carries other_programs (a routed programme, the New York CPL seeds) that the approved row lacks (round 4, RP-04).""" was = {str(v.get("state", "")).lower(): v for v in (old or {}).get("state_variations") or [] if isinstance(v, dict)} out = [] for v in e.get("state_variations") or []: if not isinstance(v, dict) or not v.get("other_programs"): continue if not (was.get(str(v.get("state", "")).lower()) or {}).get("other_programs"): out.append(v.get("state")) return out def _held_mechanisms(e, old): """The mechanism words an approved entry's code-commit version reads (its mechanism_cues and ground_not_stated words -- 'legal error', 'vacated') that the approved entry does not (the review of batch 1, round 5, RP-04: on the composed release 'CONVICTION VACATED - LEGAL ERROR' MI had answered from the approved Michigan row as a sealed conviction with no held note -- the manifest lists the words so the server drafts nothing when the record names one).""" def words(entry): out = [] for c in entry.get("mechanism_cues") or []: if isinstance(c, dict): out += [str(w) for w in c.get("words") or []] for g in entry.get("ground_not_stated") or []: if isinstance(g, dict): out += [str(w) for w in g.get("words") or []] return out have = {w.lower() for w in words(old or {})} return [w for w in dict.fromkeys(words(e)) if w.lower() not in have] def build_held_manifest(code_files, approved_files, code="", approved=""): """The held manifest for a composed tree. code_files / approved_files: {curated file name: its parsed JSON (None when absent)} at the code commit and at the approved commit.""" cd = code_files.get("us_dispositions.json") or {} ad = approved_files.get("us_dispositions.json") or {} ce = code_files.get("us_equivalency.json") or {} ae = approved_files.get("us_equivalency.json") or {} a_disp = {e.get("id"): e for e in ad.get("dispositions") or []} dispositions = [] for e in cd.get("dispositions") or []: old = a_disp.get(e.get("id")) if old is None: item = {"id": e["id"], "held": "entry", "names": list(e.get("names") or []), "aliases": list(e.get("aliases") or [])} if _added_gated(e, None): item["gated_names"] = _added_gated(e, None) if e.get("countries"): item["countries"] = list(e["countries"]) dispositions.append(item) continue names, aliases = _added(e, old, "names"), _added(e, old, "aliases") gated = _added_gated(e, old) if names or aliases or gated: item = {"id": e["id"], "held": "names", "names": names, "aliases": aliases} if gated: item["gated_names"] = gated dispositions.append(item) # round 4 (RP-04): held content on an approved entry that answers anyway -- the # acquittal-reading caveats and a routed state row; no matching words of their own caveats, rows = _held_caveats(e, old), _held_rows(e, old) if caveats: dispositions.append({"id": e["id"], "held": "caveat", "names": [], "aliases": [], "caveats": caveats}) if rows: dispositions.append({"id": e["id"], "held": "row", "names": [], "aliases": [], "states": rows}) mechanisms = _held_mechanisms(e, old) if mechanisms: # round 5 (RP-04): the readings by mechanism the approved entry lacks dispositions.append({"id": e["id"], "held": "mechanism", "names": [], "aliases": [], "words": mechanisms}) dispositions += _moved(cd, ad) a_pair = {p.get("id"): p for p in ae.get("pairings") or []} pairings = [] for p in ce.get("pairings") or []: old = a_pair.get(p.get("id")) states = p.get("state_terms") or {} if old is None: pairings.append({"id": p["id"], "held": "pairing", "us_terms": list(p.get("us_terms") or []), "state_terms": {k: list(v) for k, v in states.items()}, "negative_terms": list(p.get("negative_terms") or [])}) continue us = _added(p, old, "us_terms") was = old.get("state_terms") or {} st = {k: [t for t in v if t not in set(was.get(k) or [])] for k, v in states.items()} st = {k: v for k, v in st.items() if v} if us or st: pairings.append({"id": p["id"], "held": "terms", "us_terms": us, "state_terms": st, "negative_terms": list(p.get("negative_terms") or [])}) have = {str(k).strip().lower() for k in (ae.get("aliases") or {})} aliases = {} for code_key, v in (ce.get("aliases") or {}).items(): plain = v.get("plain") if isinstance(v, dict) else v if str(code_key).strip().lower() not in have and isinstance(plain, str) and plain.strip(): aliases[code_key] = plain return {"_about": MANIFEST_ABOUT, "code": code, "approved": approved, "dispositions": dispositions, "pairings": pairings, "aliases": aliases} def compose_drafts(wt, approved): """Put the approved commit's data/curated/drafts in the worktree's INDEX: review memos and the held batches' source drafts are held with their batch, and the review check (predeploy.check_review) holds any difference from the approved commit's subtree. Called after the composed tree's tests ran against main's files (tests/test_step1_rows.py reads them), so only the commit changes: main's drafts the approved commit lacks are left untracked in the worktree. 2026-09-21 audit: until then the subtree shipped from the code ref, and the next compose would have put the held batch-1 review memo and 138 KB of source drafts on the public Space.""" run(["git", "rm", "-r", "-q", "--cached", "--ignore-unmatch", "--", pd.DRAFTS_DIR], wt) if pd.subtree(approved, pd.DRAFTS_DIR): run(["git", "checkout", "-q", approved, "--", pd.DRAFTS_DIR], wt) return pd.subtree("HEAD", pd.DRAFTS_DIR, cwd=wt) != pd.subtree(approved, pd.DRAFTS_DIR) def write_held_manifest(wt, code, approved): """Build the manifest from the two commits' curated files and write it into the worktree; returns it.""" manifest = build_held_manifest( {n: pd.show_json(code, f"{pd.CURATED_DIR}/{n}") for n in pd.LEGAL_FILES}, {n: pd.show_json(approved, f"{pd.CURATED_DIR}/{n}") for n in pd.LEGAL_FILES}, code, approved) (wt / pd.CURATED_DIR / pd.HELD_MANIFEST).write_text( json.dumps(manifest, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") return manifest def stale_report(wt, approved): """Lines naming the approved curated pairings a corpus refresh left stale (wave 2 verifier): the composed tree's corpus (main's) against the corpus the approved table was approved with. STALE: a verify section that now states a maximum the approved pairing does not quote (the composed tree's tests fail on it -- tests/test_refresh_drift.py); RE-READ: every verify section whose text changed at all.""" eq_path = wt / pd.CURATED_DIR / "us_equivalency.json" if not eq_path.exists(): return ["stale approved content: no us_equivalency.json in the composed tree"] eq = json.loads(eq_path.read_text(encoding="utf-8")) drift, changed = pd.refresh_drift(eq, pd.corpus_reader(None, cwd=wt), pd.corpus_reader(approved)) lines = [f"STALE APPROVED CONTENT: {pid} -- {act} s. {sec} now states a maximum of " + " and ".join("life" if m >= 99 else f"{m} years" for m in new) + " that the approved pairing does not quote (a corpus refresh since approval; " "the maintainer must approve the held fix before a deploy)" for pid, act, sec, new in drift] by = {} for pid, act, sec in changed: by.setdefault(pid, []).append(f"{act} s. {sec}") if by: lines.append("RE-READ (verify sections whose text changed since the approved corpus): " + "; ".join(f"{pid}: {', '.join(secs)}" for pid, secs in sorted(by.items()))) return lines or ["stale approved content: none (no verify section changed since approval)"] def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--code-ref", default="main", help="code and corpus to ship (default main)") ap.add_argument("--approved", help="approved curated baseline (default: review_state.json)") ap.add_argument("--workdir", default=str(Path(tempfile.gettempdir()) / "canlex-compose"), help="where the detached worktree goes (removed afterwards)") ap.add_argument("--branch", default="release-composed") ap.add_argument("--no-embed", action="store_true", help="skip canlex.embed (no commit then)") ap.add_argument("--no-commit", action="store_true", help="build and test only") ap.add_argument("--keep", action="store_true", help="keep the worktree") ap.add_argument("--allow-test-failures", action="store_true") ap.add_argument("--allow-held-code", action="store_true", help="warn instead of failing when officer-facing legal text in code " "(canlex/reviewed_code.py) differs from the approved code -- only " "on the maintainer's explicit say-so") ap.add_argument("--review-state", metavar="PATH", help="compose against this review state instead of " "data/curated/review_state.json, and ship it in the composed tree " "-- a trial against another baseline (needs --no-commit)") args = ap.parse_args(argv) with contextlib.suppress(AttributeError, ValueError): sys.stdout.reconfigure(encoding="utf-8", errors="replace") if args.review_state and not args.no_commit: ap.error("--review-state composes a trial tree only: add --no-commit") state = (json.loads(Path(args.review_state).read_text(encoding="utf-8")) if args.review_state else pd.review_state()) code = pd.rev(args.code_ref) approved = pd.rev(args.approved or state["approved"]["commit"]) wt = Path(args.workdir).resolve() say(f"code {args.code_ref} = {code[:10]}; approved curated = {approved[:10]}; worktree {wt}") remove_worktree(wt) ok = False try: run(["git", "worktree", "add", "--detach", wt, code], ROOT) env = dict(os.environ, PYTHONPATH=str(wt), PYTHONIOENCODING="utf-8") _, where, _ = run([sys.executable, "-c", "import canlex; print(canlex.__file__)"], wt, env) if not Path(where.strip()).resolve().is_relative_to(wt): raise RuntimeError(f"canlex imported from {where.strip()}, not the worktree") replaced, held, restored = compose_curated(wt, code, approved, state) say(f"curated: approved version of {replaced or 'nothing (already equal)'}" + (f"; held new files left out: {held}" if held else "") + (f"; approved files restored: {restored}" if restored else "")) if args.review_state: shutil.copyfile(args.review_state, wt / pd.REVIEW_STATE) say(f"review state: {args.review_state} (a trial baseline -- not committable)") manifest = write_held_manifest(wt, code, approved) say(f"held manifest ({pd.CURATED_DIR}/{pd.HELD_MANIFEST}): " f"{len(manifest['dispositions'])} disposition entr(ies), " f"{len(manifest['pairings'])} pairing(s), {len(manifest['aliases'])} " "statute-code reading(s) held for review") for line in stale_report(wt, approved): say(line) _, text, secs = run([sys.executable, "-m", "canlex.commentary", "--allow-shrink"], wt, env) say(text.strip() + f" ({secs:.1f}s)") before = pd.show_json(code, pd.COMMENTARY) after = json.loads((wt / pd.COMMENTARY).read_text(encoding="utf-8")) changed = before != after say(f"commentary.json: {len(before)} -> {len(after)} chunks" + ("" if changed else " (unchanged)")) stale = False if changed and not args.no_embed: _, text, secs = run([sys.executable, "-m", "canlex.embed"], wt, env) say(text.strip() + f"\ncanlex.embed took {secs:.0f}s") elif changed: gap = pd.embeddings_gap(after, wt / pd.EMBEDDINGS) stale = bool(gap) say(f"--no-embed: {len(gap)} commentary chunk(s) lack a current vector" + (" -- the release will not be committed" if gap else "")) rc, text, secs = run([sys.executable, "-m", "unittest", "discover", "-s", "tests", "-q"], wt, env, check=False) summary = [l for l in text.splitlines() if l.startswith(("FAIL:", "ERROR:", "UNEXPECTED SUCCESS:", "Ran ", "OK", "FAILED"))] say("tests on the composed tree (%.0fs):\n " % secs + "\n ".join(summary)) tests_ok = rc == 0 or args.allow_test_failures if rc != 0: say("THE COMPOSED TREE FAILS ITS TESTS" + (" (--allow-test-failures)" if tests_ok else " -- not committed")) if args.no_commit or stale or not tests_ok: # (the review of batch 1, round 5: a --no-commit dry run is read from its test # summary -- with --no-embed the stale-vector count is a warning, since no commit # was asked for; without --no-embed a stale index still fails the dry run) if stale and args.no_commit and args.no_embed and rc == 0: say("--no-embed dry run: the stale vectors above are a warning only (no commit asked)") ok = args.no_commit and rc == 0 and (not stale or args.no_embed) return 0 if ok else 1 paths = ([f"{pd.CURATED_DIR}/{n}" for n in replaced + restored] + [pd.COMMENTARY, pd.EMBEDDINGS, f"{pd.CURATED_DIR}/{pd.HELD_MANIFEST}"]) if held: run(["git", "rm", "-q", "--cached", "--ignore-unmatch", "--"] + [f"{pd.CURATED_DIR}/{n}" for n in held], wt) if compose_drafts(wt, approved): say(f"{pd.DRAFTS_DIR}/: the approved commit's subtree ships (main's differs)") run(["git", "add", "--"] + paths, wt) staged, _, _ = run(["git", "diff", "--cached", "--quiet"], wt, check=False) if staged: body = (f"Code, tests and data/processed from {args.code_ref} {code}; " f"data/curated/*.json and data/curated/drafts from the approved baseline " f"{approved} (data/curated/review_state.json); commentary.json and " f"embeddings.npz regenerated from those files. Held files left out: " f"{held or 'none'}. Composed by tools/compose_release.py.") run(["git", "commit", "-q", "-m", f"Composed release: code {code[:7]} + approved curated {approved[:7]}", "-m", body], wt) release = pd.rev("HEAD", wt) # The tree checks run as the WORKTREE's tools/predeploy.py with the # worktree's canlex on the path, the way the tests did above -- in # process they would render the commentary with ROOT's canlex, the # wrong renderer for a --code-ref other than HEAD (2026-09-21 audit). rc, text, _ = run([sys.executable, "tools/predeploy.py", "--skip-tests", "--tree", release, "--approved", approved] + (["--allow-held-code"] if args.allow_held_code else []), wt, env, check=False) say(text.rstrip()) if rc != 0: say(f"release commit {release} FAILED its checks; {args.branch} not moved") return 1 pd.git("branch", "-f", args.branch, release) tree = pd.git("rev-parse", f"{release}^{{tree}}").decode().strip() say(f"\n{args.branch} -> {release}\ntree {tree}\n\nDeploy (RUNBOOK.md):\n" " git fetch space main\n" f" TREE={tree}\n" " PARENT=$(git rev-parse space/main) # the rollback point\n" ' NEW=$(git commit-tree $TREE -p $PARENT -m "Deploy: ")\n' " git branch -f deploy $NEW && git lfs ls-files -s deploy\n" " git push space deploy:main\n" " python tools/smoke_space.py") ok = True return 0 finally: if args.keep: say(f"worktree kept at {wt} (remove: git worktree remove --force {wt})") else: with contextlib.suppress(Exception): remove_worktree(wt) if not ok: say("compose_release: no release produced") if __name__ == "__main__": sys.exit(main())