Super-squash branch 'main' using huggingface_hub

Co-authored-by: Beemer <Beemer@users.noreply.huggingface.co>
Co-authored-by: claude opus 5 (1m context) <claude opus 5 (1m context)@users.noreply.huggingface.co>
Co-authored-by: Beemer <Beemer@users.noreply.huggingface.co>
Co-authored-by: claude fable 5 <claude fable 5@users.noreply.huggingface.co>

.dockerignore

@@ -0,0 +1,18 @@
+# Keep the build context small and keep secrets / regenerated files out.
+.git/
+.venv/
+venv/
+__pycache__/
+**/__pycache__/
+*.pyc
+
+# Raw downloads are not needed -- only data/processed/ is.
+data/raw/
+
+# Local-only: the CanLII key (injected as a runtime secret) and disk caches.
+canlii_key.txt
+data/citator_dbmap.json
+data/citator_cache.json
+
+# Local MCP client config; not used by the remote server.
+.mcp.json

.gitattributes

@@ -0,0 +1,4 @@
+data/processed/caselaw.json filter=lfs diff=lfs merge=lfs -text
+data/processed/embeddings.npz filter=lfs diff=lfs merge=lfs -text
+data/processed/I-3.3.json filter=lfs diff=lfs merge=lfs -text
+data/processed/I-3.3-fr.json filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+# CI for the GitHub mirror (create the mirror per RUNBOOK.md; this workflow
+# activates the moment the repo exists). Runs the offline test suite and the
+# curated-dataset schema validation on every push -- the cheap failures that
+# should never reach the production Space. The full retrieval eval needs the
+# ONNX models and LFS embeddings, so it stays a manual/scheduled job.
+name: tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  offline-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: false          # tests are offline; embeddings not needed
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # The pinned set the Space builds with: the suite imports numpy,
+      # onnxruntime, huggingface_hub and tokenizers at collection time
+      # (test_embed, test_calllog), so a shorter list errors 25 tests before
+      # they run. No model is downloaded by any test (2026-09-21 audit).
+      - name: Install the pinned dependencies (no model downloads)
+        run: pip install -r requirements.txt
+      - name: Offline test suite
+        run: python -m unittest discover -s tests -v

.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.pyc
+.venv/
+venv/
+data/raw/
+# point-in-time section pages cached by tools/pit_history.py (build-time
+# research only; already covered by data/raw/, listed so it stays ignored)
+data/raw/pit/
+canlii_key.txt
+data/citator_*.json
+US-Dispositions-Reference.pdf
+*.bundle
+data/logs/
+# derived caches rebuilt on demand (canlex/bm25.py: the BM25 postings)
+data/cache/

.mcp.json

@@ -0,0 +1,12 @@
+{
+  "mcpServers": {
+    "canlex": {
+      "command": "C:\\projects\\CanLex\\.venv\\Scripts\\python.exe",
+      "args": ["C:\\projects\\CanLex\\canlex\\server.py"],
+      "env": {
+        "CANLEX_LOG_DIR": "C:\\projects\\CanLex\\data\\logs",
+        "HF_HUB_OFFLINE": "1"
+      }
+    }
+  }
+}

Dockerfile

@@ -0,0 +1,81 @@
+# CanLex MCP server -- remote (streamable-HTTP) image.
+#
+# Builds anywhere: Hugging Face Spaces, Google Cloud Run, Fly.io, plain Docker.
+# Retrieval is fully local and key-free; the optional CanLII case citator reads
+# its key from the CANLII_API_KEY environment variable (supplied as a host
+# secret -- the key is never copied into the image).
+FROM python:3.12-slim
+
+# libgomp1 is the OpenMP runtime that onnxruntime (the reranker) links against.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends libgomp1 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Run as a non-root user (UID 1000) -- required by Hugging Face Spaces.
+RUN useradd --create-home --home-dir /app --uid 1000 app
+WORKDIR /app
+
+# Python dependencies first, so this layer caches across code changes. The
+# image installs requirements.lock: the whole set, transitive dependencies
+# included, each pinned with its sha256 hashes -- `uv pip compile` of
+# requirements.txt for Linux / Python 3.12 (RUNBOOK.md, "Audit follow-up
+# implementation (2026-09-21): group C"). Hash-checking mode refuses any
+# distribution but the locked one, so every build installs the same set (until
+# the 2026-09-21 audit only the 8 direct pins were fixed and starlette, anyio
+# and the other transitives were re-resolved at each build).
+COPY requirements.lock .
+RUN pip install --no-cache-dir --require-hashes -r requirements.lock
+
+# Application code.
+COPY --chown=app:app canlex/ ./canlex/
+
+# Pre-fetch the embedder and cross-encoder models so the cache is baked into the
+# image and the first request needs no network. The corpus is NOT re-embedded
+# here -- the committed embeddings.npz is used as-is; only the bge query model
+# (for embedding live queries) and the reranker are downloaded.
+#
+# Layer order (the code audit of 2026-09-21, optimization 9): this step reads
+# canlex/ and the network only -- Embedder() and Reranker() read nothing under
+# data/ at init -- so it runs BEFORE the corpus and curated COPYs below. Docker
+# rebuilds every layer after the first one whose input changed; after those
+# COPYs, every corpus refresh, curated change and composed release (which
+# rewrites embeddings.npz) re-ran this step and re-downloaded the 331 MB of model
+# files. Here only a change under canlex/ or to requirements.lock does. (Whether
+# the Hugging Face builder keeps layers between builds is outside our control.)
+# The user and environment it needs come first; the image's final user,
+# environment and files are what they were before the move.
+USER app
+# CANLEX_LOG_DIR turns on the local call log (canlex/calllog.py: one JSON line
+# per tool call, arguments included; loganalyze.py mines it). On the Space this
+# path is ephemeral -- lost on every restart or rebuild. The durable log is the
+# private Hugging Face dataset the server pushes to when the Space secrets
+# CANLEX_LOG_DATASET and CANLEX_LOG_TOKEN are set (RUNBOOK.md, "Call log"); the
+# HF_HUB_OFFLINE=1 below still refuses every other Hub request, model files
+# included -- calllog lets only that dataset's requests through.
+ENV HOME=/app \
+    HF_HOME=/app/.hf_cache \
+    CANLEX_HTTP=1 \
+    CANLEX_LOG_DIR=/tmp/canlex-logs \
+    PORT=7860 \
+    PYTHONUNBUFFERED=1
+
+RUN python -c "from canlex.embed import Embedder; Embedder()" \
+ && python -c "from canlex.rerank import Reranker; Reranker()"
+
+# The processed corpus (section-chunk JSON) and the prebuilt semantic
+# embeddings (embeddings.npz, committed via git-LFS). Shipping the embeddings
+# as an artifact keeps the build fast: embedding ~20k chunks on the build host
+# took ~33 min and was tripping Hugging Face's job timeout as the corpus grew.
+# Regenerate with 'python -m canlex.embed' after any corpus change and commit
+# the result.
+COPY --chown=app:app data/processed/*.json ./data/processed/
+COPY --chown=app:app data/processed/embeddings.npz ./data/processed/
+# Curated commentary datasets (the canlex_us_disposition tool reads the
+# structured source file directly, not just its processed chunks).
+COPY --chown=app:app data/curated/ ./data/curated/
+
+# From here on, model files are served from the baked cache, never fetched.
+ENV HF_HUB_OFFLINE=1
+
+EXPOSE 7860
+CMD ["python", "-m", "canlex.server"]

README.md

@@ -0,0 +1,147 @@
+---
+title: CanLex MCP
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
+# CanLex MCP
+
+A remote [Model Context Protocol](https://modelcontextprotocol.io) server for
+Canadian legal research, built for front-line border-services work. The
+legislation corpus is **bilingual**: every Act and regulation also carries its
+parallel French text, and a French-language query searches both languages
+automatically. Fourteen read-only tools over streamable HTTP (or local stdio):
+
+- **canlex_search_legislation** — hybrid (BM25 + semantic) search with a
+  cross-encoder reranker over the whole corpus: federal border, customs,
+  criminal, firearms, drug, food/health, labour and privacy legislation with
+  their Schedules (80 Justice Laws instruments, from IRPA and the Customs Act to the
+  sanctions Acts, the Criminal Records Act, the Income Tax Act and the
+  public-service labour statutes); the ENF operational manuals and IRCC program delivery
+  instructions; CBSA D-Memoranda and the AMPS Master Penalty Document; the
+  Charter; the Safe Third Country Agreement; Treasury Board collective
+  agreements; NJC directives; the CBSA Code of Conduct and the Treasury
+  Board people-management instruments; CBSA detention policy and the IRB
+  Chairperson's Guidelines and the IRB Legal Services papers on criminal
+  equivalency and criminal refusals; IRCC's rehabilitation application guide;
+  ~245 leading decisions; delegation instruments; benefit-plan booklets; and
+  CanLex's curated commentary.
+- **canlex_get_section** — fetch one provision verbatim by Act and section
+  (accepts 's. 34', '34(1)(c)', 'Schedule I').
+- **canlex_us_disposition** — step 1 of the IRPA s. 36 screening chain: is a
+  foreign disposition a "conviction"? Curated for US records, with a
+  51-jurisdiction state-by-state dataset; a non-US record (the optional
+  `country`, or a state value that is not a US state or territory) gets the
+  framework and the entry-level verdict, labelled "non-US record: curated
+  coverage is thin". The tool names keep their `us_` prefix so saved client
+  configs keep working. The answer ends with a READ NEXT block: the official
+  guidance and statute on the point, from the corpus (chunk id, status and a
+  verbatim quote; unofficial ATIP copies of withdrawn chapters flagged and
+  listed after the official sources).
+- **canlex_us_equivalency** — step 2: the Canadian equivalent of a foreign
+  offence (US-curated pairings), its verified maximum penalty, and the
+  inadmissibility branch, with the pairing's READ NEXT block.
+- **canlex_rehabilitation** — step 3: deemed rehabilitation / eligibility to
+  apply, under IRPA s. 36(3)(c) and IRPR ss. 17–18.1 — from dates, with the
+  earliest eligibility dates computed.
+- **canlex_screen_record** — the three steps in one call for a foreign record
+  (US state rows where they exist; optional `country`), from the record's
+  own words and dates: a screening memo with the draft allegation paragraph
+  (both paragraphs when the disposition or a tie is unsettled), the next
+  steps at the port of entry (s. 44 report, the removal path by paragraph,
+  TRP, rehabilitation application with the IRPR fees, family members by
+  application type), what the chain assumed, an "s. 37 also to consider"
+  block where the offence or the optional `organization` fact raises
+  organized criminality, a "Read next" line per step, and one structured
+  result line. Where the record's words reach curated content still held for
+  review that the deployment lacks, the memo says so ("HELD FOR REVIEW -- not
+  in this deployment") and lists it among what the chain assumed. A conviction in
+  Canada (country 'Canada' or a province) is screened on s. 36(1)(a)/(2)(a)
+  instead: discharges, conditional sentences, s. 18.1, the record suspension
+  as the cure, the removal path and a permanent resident's appeal line. An
+  optional `date_of_birth` (or `age_at_offence`) prints the youth block,
+  statute and IRCC policy kept apart. At the top, **THE ANSWER CHANGES WITH
+  THESE FACTS** lists each missing fact that changes the branch, the allegation
+  paragraph or the rehabilitation status (the chain re-run with every plausible
+  value; CANLEX_RESULT `decisive_missing`). An optional `conviction_date`
+  bounds the maximum at commission; `sentence_imposed` with `sentence_date`
+  adds a completion date **IF SERVED EXACTLY AS IMPOSED** (Guide 5312's rules;
+  never a verdict, and a fine blocks it); `detail: "card"` returns a result
+  card of about ten lines instead of the memo. The full memo ends with a
+  **DRAFT s. 44(1) narrative skeleton** in the ENF 5 s. 13.1 form, in English,
+  French or both (`language`): the proposed IRPA paragraph with its words, the
+  offence as recorded, the Canadian provision (the pairing's curated report
+  provision, chosen by the matched limb and the maximum at commission) and the
+  maximum in the statute's own words. Every bracket is to be verified; what
+  CanLex does not hold (name, place, the conviction date unless given, the
+  status evidence) stays `[CONFIRM: ...]`, and a result that is not settled
+  gets one allegation per reading, never a settled one.
+- **canlex_screen_history** — a whole record at once: every entry through the
+  chain (foreign or Canadian), then the record-level count for IRPR s. 18(2)
+  (equal or missing offence dates on a two-offence count give both readings),
+  deemed rehabilitation tested limb by limb (IRPR s. 18(2)(a)-(c) and s. 18.1:
+  a table with the entry that decides each limb and the date it starts or stops
+  holding; both readings where an unsettled reading decides it; an optional
+  `conviction_date` per entry, which also bounds the maximum at commission)
+  and the s. 17 clock from the most recent conviction, Canadian ones
+  included. The decisive facts, `detail: "card"` and the narrative skeleton
+  (one allegation per entry that grounds one; the two-offence limb over the
+  summary-only convictions) as for canlex_screen_record; an entry's `relief`
+  (a pardon, expungement, sealing or record suspension, or 'none') is read as
+  the disposition's later history.
+- **canlex_screen** — the same record pipeline, with the entries as typed
+  objects (one per conviction, disposition or charge, in the record's own
+  words; an unknown key is refused, so a misspelt key never silently changes
+  an answer). There is no free-text rap-sheet parsing. A record-level
+  `relief` applies to every entry without its own; 'none' confirms there is
+  none. Returns the result card by default (`detail: "full"` for everything,
+  the narrative skeleton included). Every screening tool's CANLEX_RESULT carries
+  `confidence`: what each step rests on (the entry's and pairing's authority
+  status, whether the state row was verified, how many cited decisions are in
+  the corpus, where the maximum came from, the s. 18(2) limbs left unknown).
+- **canlex_transborder** — an offence committed on entering Canada (an
+  undeclared firearm, cannabis or drugs): transborder criminality under IRPA
+  s. 36(2.1) and IRPR s. 19 — whether the offence is prescribed, whether the
+  Minister's delegate or the Immigration Division makes the deportation
+  order (IRPR ss. 228(1)(a.01), 229(1)(d.1)), no rehabilitation, the
+  contraventions check, TRP and a draft allegation. Foreign nationals only.
+- **canlex_list_acts** — the corpus inventory and its currency.
+- **canlex_whats_new** — provisions amended or newly in force since a date.
+- **canlex_citing** — reverse cross-references: what cites a given section.
+- **canlex_toc** — an Act's table of contents (Parts and marginal notes).
+- **canlex_case** — live CanLII case citator (citation graph by neutral
+  citation or URL).
+
+The server also publishes an MCP prompt (`us_record_screening`, the guided
+three-step screening flow) and resources (the corpus inventory and the
+conviction/equivalency methodology texts) that any MCP client can use.
+
+The labour side now carries the full union stack alongside the FB collective
+agreement: the CIU National By-Laws (EN/FR), CIU Regulations, Branch Rules
+and local modifications, the Branch Elections Kit, the PSAC Constitution,
+the union's member guides, and every published site VSSA — all bannered as
+bargaining-agent documents, distinct from employer policy.
+
+Retrieval runs fully locally inside the container and needs no API key. The
+optional case citator reads a CanLII key from the `CANLII_API_KEY` environment
+variable. A weekly staleness check (`canlex.refresh --check`, run every
+Monday by a Claude Code scheduled task on the maintainer's machine --
+RUNBOOK.md, "The staleness watch") watches upstream sources for drift; see
+RUNBOOK.md for the maintainer's refresh choreography.
+
+Add this Space's `/mcp` URL (`https://beemer0-canlex.hf.space/mcp`) as a
+custom connector in any Claude client, or point any other MCP-capable agent
+at it. CanLex is an MCP server only: it has no chat page of its own, and it
+calls no hosted model API.
+
+**Call log (2026-09-16):** tool calls to this server, including their
+arguments, are logged by the operator, privately, to improve CanLex
+(RUNBOOK.md, "Call log"). The server says the same to every MCP client when it
+connects (its MCP instructions) and at its root URL,
+`https://beemer0-canlex.hf.space/`.
+
+**2026-09-16:** the companion Gradio chat page (the separate `Canlex-web`
+Space) and its Google Gemini API integration were retired. The `webapp/`
+directory, its tests and the predeploy parity step were removed; the git
+history keeps them.

canlex/__init__.py

@@ -0,0 +1 @@
+"""CanLex - Canadian legal retrieval for Claude."""

canlex/_common.py

@@ -0,0 +1,503 @@
+"""Shared helpers for the ingestion modules.
+
+The ingesters grew organically and each carried private copies of the same
+plumbing; the copies were AST-diffed and consolidated here (the audit found the
+download helper in 8 near-identical copies, the line splitter in 5, and the
+whitespace collapser in 13). Deliberate per-source differences -- a politeness
+UA, a longer timeout, a different split target -- stay at the call sites as
+arguments. caselaw.py keeps its own fetch (retry/throttle for Lexum's CAPTCHA)
+and its own splitter (sentence-boundary windowing, a different algorithm).
+"""
+import base64
+import contextlib
+import http.client
+import json
+import os
+import re
+import subprocess
+import time
+import urllib.error
+import urllib.request
+from collections import Counter
+from pathlib import Path
+
+# The browser User-Agent used for government sites whose CDN denylists obvious
+# non-browser clients (the pages are public; politeness comes from caching and
+# the pauses below). The polite UA is for hosts that accept ordinary clients.
+BROWSER_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+              "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
+POLITE_UA = "CanLex/0.1"
+
+# Month-name lookup for parsing prose dates ("June 21, 2026").
+MONTHS = {m: i for i, m in enumerate(
+    ["january", "february", "march", "april", "may", "june", "july", "august",
+     "september", "october", "november", "december"], start=1)}
+
+
+# Invisible format characters (Unicode category Cf) that sources leave INSIDE
+# words: the soft hyphen U+00AD ('perform\xadance', C-46 s. 176), the zero-width
+# space, non-joiner and joiner U+200B-U+200D (after the IRB guidelines' heading
+# numbers; inside the French CITES names), the word joiner U+2060 and a stray
+# byte-order mark U+FEFF. None is whitespace, so \s never matched them: they
+# split a word into two tokens for BM25 and broke any curated verify phrase
+# spanning one. Measured on the 2026-09-21 corpus: 237 chunks carried one.
+FORMAT_CHARS = re.compile("[\u00ad\u200b\u200c\u200d\u2060\ufeff]")
+
+
+def strip_format_chars(text):
+    """`text` without the invisible in-word format characters (FORMAT_CHARS)."""
+    return FORMAT_CHARS.sub("", text or "")
+
+
+def norm_ws(text):
+    """Collapse all whitespace to single spaces and trim, after dropping the
+    invisible in-word format characters (FORMAT_CHARS, 2026-09-22 audit
+    follow-up: 'assist\\xadance' now reads 'assistance').
+
+    \\s in a Python 3 str pattern already matches NBSP, en/em/thin spaces and
+    friends, so no pre-replacement is needed (several old copies carried a
+    redundant .replace("\\xa0", " ")).
+    """
+    return re.sub(r"\s+", " ", strip_format_chars(text)).strip()
+
+
+# --- line-break hyphens in PDF text ------------------------------------------
+# A PDF's text layer keeps the typesetter's line breaks, so a word hyphenated
+# at a line end reaches the chunk as 'cross-\nexamine' and the lexical leg
+# never sees 'cross-examine' (374 PDF chunks, 2026-09-21 audit). Only 63 of
+# the 398 breaks measured have an un-hyphenated form attested elsewhere in the
+# corpus, so the hyphen is never deleted blindly: by default only the line
+# break goes ('cross-examine'), and the hyphen is dropped only when the
+# document itself uses the joined word ('immi-\ngration' where 'immigration'
+# occurs). Inside a URL the hyphen always stays ('laws-\nlois.justice.gc.ca').
+_WORD = re.compile(r"[^\W\d_]+")
+# a word and its hyphen (a printed one or a soft hyphen) closing a line
+_BREAK_TAIL = re.compile(r"([^\W\d_]+)[-\u00ad][ \t]*$")
+# the continuation that opens the next line: letters, then anything up to a space
+_BREAK_HEAD = re.compile(r"[ \t]*(([^\W\d_]+)\S*)")
+_URL_TOKEN = re.compile(r"://|^www\.|\.(?:ca|com|org|gov|gc|net|int)\b|/", re.I)
+# a soft hyphen that ends a line: the printed hyphen of a broken word
+# (WinAnsiEncoding maps the hyphen glyph to 0xAD, which pypdf reads as U+00AD)
+_SOFT_BREAK = re.compile("\u00ad(?=[ \t]*(?:\n|$))")
+
+
+def document_words(texts):
+    """Lower-cased word counts over `texts` -- a document's own vocabulary,
+    for dehyphenate."""
+    words = Counter()
+    for text in texts:
+        words.update(w.lower() for w in _WORD.findall(text or ""))
+    return words
+
+
+def mark_soft_breaks(text):
+    """A soft hyphen that ends a line of PDF text becomes '-', so the break
+    survives norm_ws (which drops every U+00AD) for dehyphenate to read."""
+    return _SOFT_BREAK.sub("-", text or "")
+
+
+def _joined(prev, tail, fragment, letters, words):
+    """`prev` (which ends in `tail` and a hyphen) with its broken word closed
+    up with `fragment` (whose leading letters are `letters`)."""
+    stem = prev.rstrip(" \t")[:-1]                 # without the hyphen
+    token = stem.split()[-1] if stem.split() else ""
+    keep = (bool(_URL_TOKEN.search(token)) or not words
+            or not letters[0].islower()
+            or not words.get((tail + letters).lower()))
+    return stem + ("-" if keep else "") + fragment
+
+
+def dehyphenate(text, words=None):
+    """`text` with each line-end hyphen break closed up (see above).
+
+    The continuation -- the next line's first run of non-space characters --
+    moves up onto the hyphenated line and the rest of that line stays where
+    it was, so line structure (and anything keyed on it) is untouched; a line
+    left empty is dropped. `words` is the document's vocabulary
+    (document_words): the hyphen goes only when the joined word is in it and
+    the continuation opens lower-case, never inside a URL. With no `words`
+    every hyphen is kept. A printed hyphen or a soft hyphen (U+00AD) counts;
+    the kept one is always '-'. Pure, for tests."""
+    out = []
+    for line in (text or "").split("\n"):
+        prev = out[-1] if out else ""
+        tail = _BREAK_TAIL.search(prev)
+        head = _BREAK_HEAD.match(line) if tail else None
+        if head:
+            out[-1] = _joined(prev, tail.group(1), head.group(1), head.group(2), words)
+            line = line[head.end():].lstrip(" \t")
+            if not line:
+                continue
+        out.append(line)
+    return "\n".join(out)
+
+
+def _ps_literal(text):
+    """`text` as a single-quoted PowerShell string literal: the only escape
+    inside single quotes is a doubled quote, so a URL or a checkout path with
+    an apostrophe (C:\\Users\\O'Brien) cannot end the literal early."""
+    return "'" + str(text).replace("'", "''") + "'"
+
+
+def powershell_download(url, dest, *, ua=BROWSER_UA, timeout=180):
+    """Fetch `url` into `dest` through PowerShell's (.NET) HTTP stack, which
+    canada.ca accepts while blocking Python's client at the TLS layer.
+
+    The script goes to PowerShell as -EncodedCommand (base64 of UTF-16LE),
+    the way refresh._powershell_probe already does, so no quoting survives
+    the Windows command line to be re-parsed; url, dest and ua are quoted
+    as PowerShell literals. Until 2026-09-21 the request was spliced into a
+    single-quoted -Command string, and one apostrophe in either was a
+    PowerShell parse error, not an HTTP error. Raises CalledProcessError or
+    TimeoutExpired; the caller decides what happens to `dest`.
+
+    When the server answered with an error status, the script writes
+    'CANLEX_HTTP_STATUS <n>' (PS_STATUS_MARKER) on a stderr line of its own
+    before rethrowing, so is_transient reads the status PowerShell got, not
+    the error page's text (verification 2026-09-22, D-2)."""
+    script = (f"$ProgressPreference = 'SilentlyContinue'\n"
+              "try {\n"
+              f"    Invoke-WebRequest -Uri {_ps_literal(url)} -OutFile {_ps_literal(dest)} "
+              f"-UseBasicParsing -UserAgent {_ps_literal(ua)}\n"
+              "} catch {\n"
+              "    if ($_.Exception.Response) {\n"
+              f"        [Console]::Error.WriteLine('{PS_STATUS_MARKER} ' + "
+              "[int]$_.Exception.Response.StatusCode.value__)\n"
+              "    }\n"
+              "    throw\n"
+              "}\n")
+    encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii")
+    subprocess.run(["powershell", "-NoProfile", "-NonInteractive",
+                    "-EncodedCommand", encoded],
+                   check=True, capture_output=True, timeout=timeout)
+
+
+def fetch_cached(url, dest, *, ua=BROWSER_UA, timeout=60, pause=0.5,
+                 force=False, powershell=False):
+    """Download a URL to `dest` once and reuse the cached bytes thereafter.
+
+    powershell=True routes the request through PowerShell's (.NET) HTTP stack,
+    which canada.ca accepts while blocking Python's client at the TLS layer.
+    `pause` is a politeness sleep after an actual download (0 to skip);
+    `timeout` applies to the urllib path (the PowerShell path keeps the fixed
+    180s subprocess budget every prior copy used).
+
+    The download lands in `<dest>.part` and is renamed into place only once
+    it succeeded, on both paths: a fetch that times out or errors leaves
+    nothing at `dest`, so the next call fetches again instead of serving a
+    truncated file as the cached copy (2026-09-21 audit: a stalled canada.ca
+    response left a 0-byte page in data/raw that every later plain run
+    reused, and a PDI page silently dropped from the corpus).
+
+    A transient failure (a 5xx, a dropped connection, a timeout) is retried
+    once (with_retry); a 404 is not, so enf.is_missing_upstream still reads
+    a retired chapter on the first answer.
+    """
+    if dest.exists() and not force:
+        return dest.read_bytes()
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    part = dest.with_name(dest.name + ".part")
+    try:
+        with_retry(lambda: _download(url, part, ua=ua, timeout=timeout,
+                                     powershell=powershell), what=url)
+        os.replace(part, dest)
+    finally:
+        with contextlib.suppress(OSError):
+            part.unlink()
+    if pause:
+        time.sleep(pause)
+    return dest.read_bytes()
+
+
+def _download(url, part, *, ua, timeout, powershell):
+    """One attempt at fetching `url` into `part` (see fetch_cached)."""
+    if powershell:
+        powershell_download(url, part, ua=ua)
+        if not part.exists():
+            raise OSError(f"{url}: PowerShell returned without writing {part.name}")
+        return
+    req = urllib.request.Request(url, headers={"User-Agent": ua})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        part.write_bytes(resp.read())
+
+
+# --- transient fetch failures ------------------------------------------------
+# One 5xx or connection reset used to fail an ingest item outright (and, with
+# the preserve-on-failure rule, pin its stale copy until the next run). Two
+# attempts in all; the pause before the second grows with the attempt number.
+# A 4xx is an answer, not a hiccup -- enf reads a 404 as IRCC retiring a
+# chapter (enf.is_missing_upstream) -- so it is never retried.
+FETCH_ATTEMPTS = 2
+FETCH_RETRY_DELAY = 5.0          # seconds before the second attempt
+# The PowerShell path's HTTP status, as powershell_download's script writes it
+# on a stderr line of its own. Until the 2026-09-22 verification (D-2) the
+# status was read as '(NNN)' anywhere in stderr, but Windows PowerShell 5.1
+# reports a failed Invoke-WebRequest as CLIXML carrying the error PAGE's text:
+# a 503 page with a body was never retried, and a 404 page quoting a phone
+# number '(514) 555-0100' was. CLIXML escapes the page's own line breaks
+# (_x000D__x000A_), so a page cannot put the marker at the start of a line.
+PS_STATUS_MARKER = "CANLEX_HTTP_STATUS"
+_PS_STATUS = re.compile(rf"^{PS_STATUS_MARKER} (\d{{3}})\r?$", re.M)
+
+
+def powershell_status(stderr):
+    """The HTTP status the PowerShell fetch's marker line reports
+    (PS_STATUS_MARKER), or None when the stderr carries no marker (no HTTP
+    answer, or a script that predates the marker). Only the marker line is
+    read, never the error page's own text."""
+    if isinstance(stderr, bytes):
+        stderr = stderr.decode("utf-8", "replace")
+    m = _PS_STATUS.search(stderr or "")
+    return int(m.group(1)) if m else None
+
+
+# .NET's own words for a failure with no HTTP answer (so no page, and no marker)
+_PS_TRANSIENT = re.compile(r"timed out|timeout|unable to connect|connection was closed"
+                           r"|forcibly closed|temporar", re.I)
+
+
+def is_transient(exc):
+    """True when a fetch failure is worth one more try: an HTTP 5xx, a
+    URLError without a status (DNS, refused, reset), a timeout, a dropped
+    connection, a truncated body, or a PowerShell failure whose status
+    marker (PS_STATUS_MARKER) is a 5xx or which got no HTTP answer and
+    reports a connection-level error. Never a 4xx, and never a status read
+    from the error page's text. Pure, for tests."""
+    if isinstance(exc, urllib.error.HTTPError):
+        return exc.code >= 500
+    if isinstance(exc, (urllib.error.URLError, subprocess.TimeoutExpired,
+                        TimeoutError, ConnectionError, http.client.IncompleteRead)):
+        return True
+    if isinstance(exc, subprocess.CalledProcessError):
+        stderr = exc.stderr or ""
+        if isinstance(stderr, bytes):
+            stderr = stderr.decode("utf-8", "replace")
+        status = _PS_STATUS.search(stderr)
+        if status:
+            return status.group(1).startswith("5")
+        return bool(_PS_TRANSIENT.search(stderr))
+    return False
+
+
+def with_retry(fetch, *, what="", attempts=None, delay=None):
+    """fetch() -- again after a pause when it fails transiently (is_transient),
+    up to FETCH_ATTEMPTS attempts in all. The last failure, or any
+    non-transient one, propagates unchanged."""
+    attempts = FETCH_ATTEMPTS if attempts is None else attempts
+    delay = FETCH_RETRY_DELAY if delay is None else delay
+    for attempt in range(1, attempts + 1):
+        try:
+            return fetch()
+        except Exception as exc:
+            if attempt >= attempts or not is_transient(exc):
+                raise
+            print(f"  retrying {what or 'fetch'} in {delay * attempt:g}s "
+                  f"after {type(exc).__name__}: {str(exc)[:80]}")
+            time.sleep(delay * attempt)
+    return None
+
+
+# Windows refuses to rename onto a file another process holds open without
+# FILE_SHARE_DELETE -- Python's own open(), most scanners, indexers and
+# editors -- with [WinError 5] Access is denied, where the plain write_text
+# the atomic helpers replaced succeeds (verification 2026-09-22, D-3). Such a
+# hold is usually brief, so the rename is tried REPLACE_ATTEMPTS times in all,
+# REPLACE_RETRY_DELAY apart, before the PermissionError propagates.
+REPLACE_ATTEMPTS = 5
+REPLACE_RETRY_DELAY = 0.2        # seconds between rename attempts
+
+
+def _replace(tmp, path):
+    """os.replace(tmp, path), retried on PermissionError (REPLACE_ATTEMPTS
+    in all); the last PermissionError, or any other OSError, propagates."""
+    for attempt in range(1, REPLACE_ATTEMPTS + 1):
+        try:
+            os.replace(tmp, path)
+            return
+        except PermissionError:
+            if attempt >= REPLACE_ATTEMPTS:
+                raise
+            time.sleep(REPLACE_RETRY_DELAY)
+
+
+def atomic_write_text(path, text, *, encoding="utf-8"):
+    """Write `text` to `path` through a sibling temporary file renamed into
+    place, so a run killed mid-write leaves the old file (or none), never a
+    truncated one. For the small marker and cache files the ingesters and
+    the staleness watch keep (the guidance last-seen marker, the CIU
+    decisions date, the raw case-law pages). A rename refused because
+    another process briefly holds `path` open is retried (_replace)."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+    try:
+        tmp.write_text(text, encoding=encoding)
+        _replace(tmp, path)
+    finally:
+        with contextlib.suppress(OSError):
+            tmp.unlink()
+    return path
+
+
+def atomic_write_bytes(path, data):
+    """atomic_write_text for bytes (the Justice Laws XML cache)."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
+    try:
+        tmp.write_bytes(data)
+        _replace(tmp, path)
+    finally:
+        with contextlib.suppress(OSError):
+            tmp.unlink()
+    return path
+
+
+def split_lines(text, target=1800, fold_stubs=False):
+    """Split text into ~target-sized pieces at line boundaries.
+
+    fold_stubs=True additionally folds a small (<200-char) leading or trailing
+    remnant into its neighbour so a section never yields a stub piece -- the
+    behaviour ingest.py and benefits.py always had. The other callers
+    deliberately keep fold_stubs=False: enabling it would change their chunk
+    boundaries, which is an eval-gated change, not a refactor.
+    """
+    if len(text) <= target:
+        return [text]
+    pieces, buf, size = [], [], 0
+    for line in text.split("\n"):
+        if size + len(line) > target and buf:
+            pieces.append("\n".join(buf))
+            buf, size = [], 0
+        buf.append(line)
+        size += len(line) + 1
+    if buf:
+        pieces.append("\n".join(buf))
+    if fold_stubs:
+        if len(pieces) > 1 and len(pieces[-1]) < 200:
+            pieces[-2] = pieces[-2] + "\n" + pieces[-1]
+            pieces.pop()
+        if len(pieces) > 1 and len(pieces[0]) < 200:
+            pieces[1] = pieces[0] + "\n" + pieces[1]
+            pieces.pop(0)
+    return pieces
+
+
+# Every ingester rebuilds its whole processed file from a fresh scrape, so an
+# upstream page that changes shape -- or a run that loses its network halfway --
+# shows up as a collapse in chunk count, and the write turns that into data
+# loss. On 2026-07-27 CBSA's rebuilt D-memo index scraped to zero memos and the
+# write replaced 1,870 good chunks with an empty list; one line of stdout was
+# the only warning. Refuse to write below this share of what is already stored.
+CORPUS_SHRINK_GUARD = 0.8
+
+
+def stored_chunks(path):
+    """The chunks already written to `path`, or [] if absent or unreadable."""
+    if not path.exists():
+        return []
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (ValueError, OSError):
+        return []
+
+
+def safe_to_write(new_count, old_count, ratio=CORPUS_SHRINK_GUARD):
+    """False when writing `new_count` chunks would gut an existing corpus.
+
+    Pure, for tests. The write is the irreversible step, so it is the one to
+    gate -- a first-ever run (old_count 0) is always allowed.
+    """
+    return old_count == 0 or new_count >= old_count * ratio
+
+
+def preserve_dropped(chunks, stored, *, key=lambda c: c["section"], only=None):
+    """Re-attach the last-good chunks of items this run failed to produce.
+
+    `key` maps a chunk to the item it belongs to; `only` restricts preservation
+    to a set of item identities (pass the ones that actually errored, so items
+    dropped ON PURPOSE -- cancelled memos, retired manuals -- are not
+    resurrected). Returns (chunks, preserved identities).
+    """
+    def identity(chunk):
+        # A chunk written by an older schema may not carry the identity field
+        # at all. It simply is not preservable -- but crashing here would take
+        # down the rebuild at the exact moment the guard exists to protect it.
+        try:
+            return key(chunk)
+        except (KeyError, IndexError, TypeError):
+            return None
+
+    fresh = {identity(c) for c in chunks}
+    by_item = {}
+    for chunk in stored:
+        item = identity(chunk)
+        if item is not None:
+            by_item.setdefault(item, []).append(chunk)
+    dropped = sorted(item for item in by_item
+                     if item not in fresh and (only is None or item in only))
+    for item in dropped:
+        chunks.extend(by_item[item])
+    return chunks, dropped
+
+
+def write_corpus(path, chunks, *, indent=2, allow_shrink=False,
+                 ratio=CORPUS_SHRINK_GUARD):
+    """Write `chunks` to `path`, unless that would gut the stored corpus.
+
+    Returns True if written, False if refused -- callers exit non-zero on False
+    so a broken scrape fails loudly instead of quietly emptying a file.
+    """
+    stored = stored_chunks(path)
+    if not allow_shrink and not safe_to_write(len(chunks), len(stored), ratio):
+        print(f"  REFUSING to write: {len(chunks)} chunks would replace "
+              f"{len(stored)} already in {path.name}. The upstream source "
+              f"likely changed shape -- check the scrape before writing "
+              f"(re-run with --allow-shrink once the drop is understood).")
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # Write-then-rename, not write-in-place: a run killed mid-write (Ctrl-C on
+    # the scheduled refresh, a full disk) would otherwise leave truncated JSON
+    # behind, and stored_chunks reads unparseable as empty -- which disables
+    # the guard above on the very next run, when it is needed most.
+    tmp = path.with_name(path.name + ".tmp")
+    tmp.write_text(json.dumps(chunks, ensure_ascii=False, indent=indent),
+                   encoding="utf-8")
+    os.replace(tmp, path)
+    return True
+
+
+def dup_suffix(n):
+    """The letter suffix for the n-th repeat of an id (n >= 1).
+
+    Bijective base 26 over a..z on n + 1: 1 -> 'b', ..., 25 -> 'z', then
+    26 -> 'aa', 27 -> 'ab', ... 'az', 'ba', ... The first 25 are exactly the
+    single letters uniquify_ids has always produced, so every id already in a
+    corpus keeps its spelling; only repeats past 'z' change, and those used to
+    run on into '{', '|', DEL and C1 control characters (chr(ord('a') + n))."""
+    if n < 1:
+        raise ValueError("a first occurrence takes no suffix")
+    k, out = n + 1, ""
+    while k:
+        k, rem = divmod(k - 1, 26)
+        out = chr(ord("a") + rem) + out
+    return out
+
+
+def uniquify_ids(chunks):
+    """Rename duplicate chunk ids in place (second occurrence -> id-b, -c...,
+    -z, then -aa, -ab, ...; see dup_suffix).
+
+    Duplicate ids silently orphan chunks downstream: the embeddings loader
+    keys rows by id, so one twin gets vectors and the other degrades to
+    BM25-only with nothing but a stderr note. Generators whose source
+    numbering can repeat (a manual's TOC line and body heading both reading
+    '5.1'; delegation items re-using an instrument number) run their output
+    through this before writing."""
+    seen = {}
+    for c in chunks:
+        n = seen.get(c["id"], 0)
+        seen[c["id"]] = n + 1
+        if n:
+            c["id"] = f"{c['id']}-{dup_suffix(n)}"
+    return chunks

canlex/agreement.py

@@ -0,0 +1,242 @@
+"""Ingest Treasury Board collective agreements (HTML) into Article-level chunks.
+
+A collective agreement is a binding contract between the Treasury Board and a
+bargaining agent for one occupational group. Chunks are tagged
+doc_type="agreement" so CanLex keeps them distinct from legislation and guidance.
+"""
+import re
+import sys
+
+from bs4 import BeautifulSoup
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, write_corpus)
+from .config import RAW_DIR, PROCESSED_DIR
+
+AGREEMENT_DIR = RAW_DIR / "agreements"
+OUT_FILE = PROCESSED_DIR / "agreements.json"
+
+# Treasury Board collective agreements to ingest. Add an entry to ingest more.
+AGREEMENTS = {
+    "FB": {
+        "short": "FB Agreement",
+        "name": "FB Group Collective Agreement (Border Services)",
+        "cite": "FB Collective Agreement",
+        "url": "https://www.canada.ca/en/treasury-board-secretariat/topics/pay/"
+               "collective-agreements/fb.html",
+        # TBS publishes the agreement bilingually; the French edition is the
+        # lang='fr' twin (agreements-fr.json), reachable by French queries only.
+        "url_fr": "https://www.canada.ca/fr/secretariat-conseil-tresor/sujets/"
+                  "remuneration/conventions-collectives/fb.html",
+        "cite_fr": "Convention collective FB",
+    },
+}
+OUT_FILE_FR = OUT_FILE.parent / "agreements-fr.json"
+
+_SKIP_HEADINGS = {"table of contents", "note to readers", "page details",
+                  "on this page", "table des matières", "note aux lecteurs",
+                  "détails de la page", "sur cette page"}
+_CONTENT_TAGS = {"p", "dl", "table", "h4", "h5", "h6", "blockquote"}
+_ARTICLE = re.compile(r"Article\s+(\S+?)\s*[:–-]\s*(.+)", re.I)
+_APPENDIX = re.compile(r"(?:Appendix|Annexe)\b", re.I)
+
+
+def _fetch(url, dest, force=False):
+    """Download a page. canada.ca blocks Python's HTTP client at the TLS layer,
+    so fetch via PowerShell's (.NET) HTTP stack, which the site accepts."""
+    return fetch_cached(url, dest, force=force, powershell=True)
+
+
+_PIECE_TARGET = 1800   # keep pieces under the embedder's ~2000-char window
+
+
+def _split(text, target=_PIECE_TARGET):
+    """Split long text into ~target-sized pieces at line boundaries."""
+    return split_lines(text, target)
+
+
+def _block_text(heading):
+    """Readable text from a heading up to the next h2/h3 (sections unwrapped)."""
+    lines = []
+    for sib in heading.find_next_siblings():
+        if sib.name in ("h2", "h3"):
+            break
+        if sib.name in ("ul", "ol"):
+            for li in sib.find_all("li", recursive=False):
+                item = _norm(li.get_text(" ", strip=True))
+                if item:
+                    lines.append(f"- {item}")
+        elif sib.name in _CONTENT_TAGS:
+            text = _norm(sib.get_text(" ", strip=True))
+            if text:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def parse_agreement(html, code, lang="en"):
+    """Parse a collective agreement page into one chunk per Article / Appendix.
+    lang='fr' parses the French edition: same article numbers, French titles
+    and citation, '-fr-' id infix and lang='fr' on every chunk."""
+    meta = AGREEMENTS[code]
+    cite = meta["cite_fr"] if lang == "fr" else meta["cite"]
+    url = meta["url_fr"] if lang == "fr" else meta["url"]
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+
+    # The first <time datetime> is the agreement's expiry; the dateModified one
+    # is the page's date.
+    expiry = ""
+    for t in main.find_all("time"):
+        if t.get("datetime") and t.get("property") != "dateModified":
+            expiry = _norm(t.get("datetime"))
+            break
+
+    for tag in main.find_all(["section", "div"]):
+        tag.unwrap()  # flatten so each heading and its content become siblings
+
+    chunks = []
+    current_part = ""
+    sec_idx = 0
+    for h in main.find_all(["h2", "h3"]):
+        if "wb-inv" in (h.get("class") or []):
+            continue
+        text = _norm(h.get_text(" ", strip=True)).lstrip("*").strip()
+        if not text or text.lower() in _SKIP_HEADINGS:
+            continue
+        if h.name == "h2":
+            current_part = text
+        body = _block_text(h)
+        if not body:
+            continue
+
+        article = _ARTICLE.match(text)
+        if article:
+            number = article.group(1).rstrip(":.")
+            note = article.group(2).strip()
+            citation = (f"{cite}, article {number}" if lang == "fr"
+                        else f"{cite}, Article {number}")
+        elif _APPENDIX.match(text):
+            number, note = text, ""
+            citation = f"{cite}, {text}"
+        else:
+            # Memoranda of Agreement/Understanding have long titles: keep them in
+            # the note only, so the rendered header does not repeat them.
+            number, note = "", text
+            citation = cite
+
+        sec_idx += 1
+        # Split a long article so each piece fits the embed window; pieces share
+        # the base id and take a '-p<k>' suffix so the diversity cap groups them.
+        pieces = _split(body)
+        for k, piece in enumerate(pieces, start=1):
+            base = (f"agreement-fr-{code}-{sec_idx}" if lang == "fr"
+                    else f"agreement-{code}-{sec_idx}")
+            chunk_id = base if len(pieces) == 1 else f"{base}-p{k}"
+            pnote = note if len(pieces) == 1 else (
+                f"{note} (partie {k})" if lang == "fr" else f"{note} (part {k})").strip()
+            chunks.append({
+                "id": chunk_id,
+                "doc_type": "agreement",
+                "act_code": code,
+                "act_short": meta["short"],
+                "act_name": meta["name"],
+                "section": number,
+                "marginal_note": pnote,
+                "part": current_part if h.name == "h3" else "",
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": "",
+                "current_to": expiry,
+                "citation": citation,
+                "source_url": url,
+                **({"lang": "fr"} if lang == "fr" else {}),
+            })
+    return chunks
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of agreements this run could not rebuild.
+
+    Ingestion is a full rebuild, so an agreement whose page fails to fetch --
+    or whose canada.ca layout changes so that nothing parses out of it --
+    disappears from the corpus. An agreement that is still in force is better
+    served from the copy we already had than not at all. Restricted to the
+    codes that actually failed, so one removed from AGREEMENTS on purpose is
+    not resurrected. Returns (chunks, preserved codes).
+    """
+    failed = {code for code, _why in failures}
+    return preserve_dropped(chunks, stored, key=lambda c: c["act_code"],
+                            only=failed)
+
+
+def ingest(force=False, allow_shrink=False):
+    all_chunks, failures = [], []
+    for code, meta in AGREEMENTS.items():
+        print(f"Ingesting {code} ({meta['short']})...")
+        try:
+            html = _fetch(meta["url"], AGREEMENT_DIR / f"{code}.html", force=force)
+            chunks = parse_agreement(html, code)
+            print(f"  {len(chunks)} chunks")
+            all_chunks.extend(chunks)
+            if not chunks:
+                # A page that parses to nothing is a redesign upstream, not an
+                # empty agreement -- same class of failure as a dead fetch.
+                failures.append((code, "no content parsed"))
+        except Exception as exc:
+            failures.append((code, f"{type(exc).__name__}: {exc}"))
+            print(f"  FAILED {code}: {type(exc).__name__}: {exc}")
+
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT_FILE))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} agreement(s): "
+              f"{', '.join(preserved)}")
+    if not write_corpus(OUT_FILE, all_chunks, indent=2,
+                        allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} chunks from {len(AGREEMENTS)} agreement(s) -> {OUT_FILE.name}")
+    return True
+
+
+def ingest_fr(force=False, allow_shrink=False):
+    """The French editions -> agreements-fr.json (same shape as ingest)."""
+    all_chunks, failures = [], []
+    for code, meta in AGREEMENTS.items():
+        if not meta.get("url_fr"):
+            continue
+        print(f"Ingesting {code} (French edition)...")
+        try:
+            html = _fetch(meta["url_fr"], AGREEMENT_DIR / f"{code}-fr.html",
+                          force=force)
+            chunks = parse_agreement(html, code, lang="fr")
+            print(f"  {len(chunks)} chunks")
+            all_chunks.extend(chunks)
+            if not chunks:
+                failures.append((code, "no content parsed"))
+        except Exception as exc:
+            failures.append((code, f"{type(exc).__name__}: {exc}"))
+            print(f"  FAILED {code}: {type(exc).__name__}: {exc}")
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT_FILE_FR))
+    if preserved:
+        print(f"  kept the last-good copy of: {', '.join(preserved)}")
+    if not write_corpus(OUT_FILE_FR, all_chunks, indent=2,
+                        allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} French chunks -> {OUT_FILE_FR.name}")
+    return True
+
+
+def main():
+    run = ingest_fr if "--fr" in sys.argv else ingest
+    ok = run(force="--force" in sys.argv,
+             allow_shrink="--allow-shrink" in sys.argv)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/airports.py

@@ -0,0 +1,237 @@
+"""Aerodromes by the names officers use (2026-10-15).
+
+The Canadian Aviation Security Regulations, 2012 (SOR/2011-318) list the 89
+aerodromes of Schedules 1-3 by city and ICAO location indicator only ('Vancouver |
+CYVR'), so a search for 'YVR', 'Pearson' or 'Billy Bishop' never met the row that
+answers it. airport_terms() reads an IATA code or an airport's name in a query and
+returns the Schedules' own words for each aerodrome named ('CYVR Vancouver'), which
+LegislationIndex.search appends for its recall stages. The statute text is never
+changed: this table only decides which schedule row a query can reach.
+
+Sources. The ICAO indicator, the city as written and the class are the Regulations'
+own (Schedules 1-3, consolidation current to 2025-04-14); tests/test_airports.py holds
+this table to the ingested schedules, so an amendment to them fails the test until the
+table follows. The IATA codes and the airports' names: Wikidata (CC0), retrieved
+2026-10-15, every code and name checked against a second source the same day.
+
+Three kinds of words reach an aerodrome:
+ - its IATA code, as a whole word in either case -- except the codes that are also
+   words ('yam', 'yay', 'yow'), which count only in capitals;
+ - a DISTINCTIVE name, on its own ('Billy Bishop', 'Toronto Pearson', 'CFB Gander');
+ - a city or a name that also means a person or a place ('Pearson', 'Trudeau',
+   'Regina'), only beside an airport word ('Pearson airport', 'aéroport de Québec',
+   'is Pearson class 1', 'Calgary International').
+A city alone never counts, and a more specific name beats the city it contains:
+'Toronto Pearson airport' reaches CYYZ, not the other two Toronto aerodromes.
+"""
+import re
+import unicodedata
+from typing import NamedTuple
+
+
+class Aerodrome(NamedTuple):
+    icao: str
+    iata: str
+    city: str          # as Schedules 1-3 write it
+    cls: int           # the Schedule: 1, 2 or 3
+    places: tuple = ()     # how a query names the place; counts beside an airport word
+    names: tuple = ()      # the airport's own names that need an airport word too
+    distinct: tuple = ()   # names that mean this aerodrome on their own
+
+
+AERODROMES = (
+    # --- Schedule 1: Class 1 aerodromes ---
+    Aerodrome("CYYC", "YYC", "Calgary", 1, ("calgary",)),
+    Aerodrome("CYEG", "YEG", "Edmonton", 1, ("edmonton",)),
+    Aerodrome("CYHZ", "YHZ", "Halifax", 1, ("halifax",),
+              ("stanfield", "robert l stanfield"), ("halifax stanfield",)),
+    Aerodrome("CYMX", "YMX", "Montréal", 1, ("montreal",), ("mirabel",),
+              ("montreal mirabel",)),
+    Aerodrome("CYUL", "YUL", "Montréal", 1, ("montreal",),
+              ("trudeau", "pierre elliott trudeau", "dorval"), ("montreal trudeau",)),
+    Aerodrome("CYOW", "YOW", "Ottawa", 1, ("ottawa",), ("macdonald cartier",),
+              ("ottawa macdonald cartier",)),
+    Aerodrome("CYYZ", "YYZ", "Toronto", 1, ("toronto",),
+              ("pearson", "lester b pearson"), ("toronto pearson",)),
+    Aerodrome("CYVR", "YVR", "Vancouver", 1, ("vancouver",)),
+    Aerodrome("CYWG", "YWG", "Winnipeg", 1, ("winnipeg",),
+              ("richardson", "james armstrong richardson", "j a richardson"),
+              ("winnipeg richardson", "cfb winnipeg", "17 wing")),
+    # --- Schedule 2: Class 2 aerodromes ---
+    Aerodrome("CYYG", "YYG", "Charlottetown", 2, ("charlottetown",),
+              ("alexander b campbell",)),    # renamed May 2026
+    Aerodrome("CYFC", "YFC", "Fredericton", 2, ("fredericton",)),
+    Aerodrome("CYQX", "YQX", "Gander", 2, ("gander",), (), ("cfb gander", "9 wing")),
+    Aerodrome("CYFB", "YFB", "Iqaluit", 2, ("iqaluit",), ("frobisher bay",)),
+    Aerodrome("CYLW", "YLW", "Kelowna", 2, ("kelowna",)),
+    Aerodrome("CYXU", "YXU", "London", 2, ("london ontario", "london ont", "london on")),
+    Aerodrome("CYQM", "YQM", "Moncton", 2, ("moncton",),
+              ("romeo leblanc",), ("greater moncton",)),    # Roméo LeBlanc since 2016
+    Aerodrome("CYXS", "YXS", "Prince George", 2, ("prince george",)),
+    Aerodrome("CYQB", "YQB", "Québec", 2, ("quebec", "quebec city", "ville de quebec"),
+              ("jean lesage",)),
+    Aerodrome("CYQR", "YQR", "Regina", 2, ("regina",)),
+    Aerodrome("CYSJ", "YSJ", "Saint John", 2, ("saint john",)),
+    Aerodrome("CYYT", "YYT", "St. John’s", 2, ("st john's", "st johns", "saint john's")),
+    Aerodrome("CYXE", "YXE", "Saskatoon", 2, ("saskatoon",),
+              ("diefenbaker", "john g diefenbaker")),
+    Aerodrome("CYSB", "YSB", "Sudbury", 2, ("sudbury", "greater sudbury")),
+    Aerodrome("CYQT", "YQT", "Thunder Bay", 2, ("thunder bay",)),
+    Aerodrome("CYTZ", "YTZ", "Toronto", 2, ("toronto",),
+              ("toronto island", "toronto city", "toronto city centre"),
+              ("billy bishop",)),
+    Aerodrome("CYYJ", "YYJ", "Victoria", 2, ("victoria",)),
+    Aerodrome("CYXY", "YXY", "Whitehorse", 2, ("whitehorse",), (), ("erik nielsen",)),
+    Aerodrome("CYQG", "YQG", "Windsor", 2, ("windsor",)),
+    Aerodrome("CYZF", "YZF", "Yellowknife", 2, ("yellowknife",)),
+    # --- Schedule 3: Class 3 aerodromes ---
+    Aerodrome("CYXX", "YXX", "Abbotsford", 3, ("abbotsford",)),
+    Aerodrome("CYTF", "YTF", "Alma", 3, ("alma",)),
+    Aerodrome("CYBG", "YBG", "Bagotville", 3, ("bagotville",), ("saguenay",),
+              ("cfb bagotville", "3 wing")),
+    Aerodrome("CYBC", "YBC", "Baie-Comeau", 3, ("baie comeau",)),
+    Aerodrome("CZBF", "ZBF", "Bathurst", 3, ("bathurst",)),
+    Aerodrome("CYBR", "YBR", "Brandon", 3, ("brandon",)),
+    Aerodrome("CYBL", "YBL", "Campbell River", 3, ("campbell river",)),
+    Aerodrome("CYCG", "YCG", "Castlegar", 3, ("castlegar",), ("west kootenay",)),
+    Aerodrome("CYCL", "YCL", "Charlo", 3, ("charlo",)),
+    Aerodrome("CYMT", "YMT", "Chibougamau/Chapais", 3, ("chibougamau", "chapais")),
+    Aerodrome("CZUM", "ZUM", "Churchill Falls", 3, ("churchill falls",)),
+    Aerodrome("CYQQ", "YQQ", "Comox", 3, ("comox", "comox valley"), (),
+              ("cfb comox", "19 wing")),
+    Aerodrome("CYXC", "YXC", "Cranbrook", 3, ("cranbrook",), ("canadian rockies",)),
+    Aerodrome("CYDQ", "YDQ", "Dawson Creek", 3, ("dawson creek",)),
+    Aerodrome("CYDF", "YDF", "Deer Lake", 3, ("deer lake",)),
+    Aerodrome("CYMM", "YMM", "Fort McMurray", 3, ("fort mcmurray",)),
+    Aerodrome("CYXJ", "YXJ", "Fort St. John", 3, ("fort st john", "fort saint john"),
+              ("north peace",)),
+    Aerodrome("CYGP", "YGP", "Gaspé", 3, ("gaspe",), ("michel pouliot",)),
+    Aerodrome("CYYR", "YYR", "Goose Bay", 3, ("goose bay", "happy valley goose bay"), (),
+              ("cfb goose bay", "5 wing")),
+    Aerodrome("CYQU", "YQU", "Grande Prairie", 3, ("grande prairie",)),
+    Aerodrome("CYHM", "YHM", "Hamilton", 3, ("hamilton",), ("munro", "john c munro")),
+    Aerodrome("CYGR", "YGR", "Îles-de-la-Madeleine", 3,
+              ("iles de la madeleine", "magdalen islands")),
+    Aerodrome("CYKA", "YKA", "Kamloops", 3, ("kamloops",)),
+    Aerodrome("CYGK", "YGK", "Kingston", 3, ("kingston ontario", "kingston ont", "kingston on"),
+              ("norman rogers",)),
+    Aerodrome("CYKF", "YKF", "Kitchener/Waterloo", 3,
+              ("kitchener", "waterloo", "kitchener waterloo"), (), ("region of waterloo",)),
+    Aerodrome("CYVP", "YVP", "Kuujjuaq", 3, ("kuujjuaq",)),
+    Aerodrome("CYGW", "YGW", "Kuujjuarapik", 3, ("kuujjuarapik",)),
+    Aerodrome("CYGL", "YGL", "La Grande Rivière", 3,
+              ("la grande riviere", "radisson", "grande riviere")),
+    Aerodrome("CYFJ", "YTM", "La Macaza / Mont-Tremblant", 3, ("la macaza", "mont tremblant")),
+    Aerodrome("CYQL", "YQL", "Lethbridge", 3, ("lethbridge", "lethbridge county")),
+    Aerodrome("CYLL", "YLL", "Lloydminster", 3, ("lloydminster",)),
+    Aerodrome("CYBX", "YBX", "Lourdes-de-Blanc-Sablon", 3,
+              ("lourdes de blanc sablon", "blanc sablon")),
+    Aerodrome("CYXH", "YXH", "Medicine Hat", 3, ("medicine hat",)),
+    Aerodrome("CYYY", "YYY", "Mont-Joli", 3, ("mont joli",)),
+    Aerodrome("CYCD", "YCD", "Nanaimo", 3, ("nanaimo",)),
+    Aerodrome("CYYB", "YYB", "North Bay", 3, ("north bay",), ("jack garland",)),
+    Aerodrome("CYYF", "YYF", "Penticton", 3, ("penticton",)),
+    Aerodrome("CYPA", "YPA", "Prince Albert", 3, ("prince albert",), ("glass field",)),
+    Aerodrome("CYPR", "YPR", "Prince Rupert", 3, ("prince rupert",)),
+    Aerodrome("CYQZ", "YQZ", "Quesnel", 3, ("quesnel",)),
+    Aerodrome("CYQF", "YQF", "Red Deer", 3, ("red deer",)),
+    Aerodrome("CYRJ", "YRJ", "Roberval", 3, ("roberval",)),
+    Aerodrome("CYUY", "YUY", "Rouyn-Noranda", 3, ("rouyn noranda",)),
+    Aerodrome("CYAY", "YAY", "St. Anthony", 3, ("st anthony", "saint anthony")),
+    Aerodrome("CYSL", "YSL", "Saint-Léonard", 3, ("saint leonard", "st leonard")),
+    Aerodrome("CYZP", "YZP", "Sandspit", 3, ("sandspit",), (),
+              ("k'il kun xidgwangs daanaay",)),
+    Aerodrome("CYZR", "YZR", "Sarnia", 3, ("sarnia",), ("chris hadfield",)),
+    Aerodrome("CYAM", "YAM", "Sault Ste. Marie", 3, ("sault ste marie", "sault sainte marie")),
+    Aerodrome("CYZV", "YZV", "Sept-Îles", 3, ("sept iles",)),
+    Aerodrome("CYYD", "YYD", "Smithers", 3, ("smithers",)),
+    Aerodrome("CYJT", "YJT", "Stephenville", 3, ("stephenville",)),
+    Aerodrome("CYQY", "YQY", "Sydney", 3, ("sydney nova scotia", "sydney ns", "sydney n s"),
+              ("mccurdy", "douglas mccurdy", "j a douglas mccurdy")),
+    Aerodrome("CYXT", "YXT", "Terrace", 3, ("terrace",), ("northwest regional",)),
+    Aerodrome("CYTH", "YTH", "Thompson", 3, ("thompson",)),
+    Aerodrome("CYTS", "YTS", "Timmins", 3, ("timmins",), ("victor m power",)),
+    # Buttonville closed on 2023-11-30 and IATA has withdrawn YKZ; the Regulations
+    # still list it, so its name still reaches the row
+    Aerodrome("CYKZ", "", "Toronto", 3, ("toronto",), (), ("buttonville",)),
+    Aerodrome("CYVO", "YVO", "Val-d’Or", 3, ("val d'or",)),
+    Aerodrome("CYWK", "YWK", "Wabush", 3, ("wabush",)),
+    Aerodrome("CYWL", "YWL", "Williams Lake", 3, ("williams lake",)),
+    Aerodrome("CYQI", "YQI", "Yarmouth", 3, ("yarmouth",)),
+)
+
+# IATA codes that are also words an officer may type ('can I import a yam'): they
+# count only as written in capitals.
+_WORD_CODES = frozenset({"YAM", "YAY", "YOW"})
+
+# Words that make a place or a person's name mean an airport. 'International',
+# 'Regional' and 'Municipal' count only right after the name ('Calgary International',
+# 'North Peace Regional'): 'Vancouver international students' is not an airport.
+# 'Terminal' is not one ('Vancouver cruise terminal').
+_CONTEXT = re.compile(r"(?<![\w'])(?:airports?|aerodromes?|airfields?|aeroports?|aerogares?"
+                      r"|class [123]|classe [123]|categorie [123])(?![\w'])")
+_INTERNATIONAL = re.compile(r"\s*(?:internationale?|regionale?|municipale?)(?![\w'])")
+_WINDOW = 3     # words either side of a name in which an airport word must sit
+
+
+def _fold(text):
+    """Lower case, accents and curly quotes folded, hyphens, slashes and other
+    punctuation as spaces ('Montréal-Trudeau' -> 'montreal trudeau', 'St. John’s'
+    -> "st john's")."""
+    text = unicodedata.normalize("NFKD", text or "")
+    text = "".join(ch for ch in text if not unicodedata.combining(ch)).lower()
+    text = text.replace("’", "'").replace("‘", "'")
+    return " ".join(re.sub(r"[^\w']+", " ", text).split())
+
+
+def _phrase(words):
+    return re.compile(r"(?<![\w'])" + re.escape(words) + r"(?![\w'])")
+
+
+_ENTRIES = tuple(
+    (a, tuple(_phrase(p) for p in a.distinct),
+     tuple(_phrase(p) for p in a.names), tuple(_phrase(p) for p in a.places))
+    for a in AERODROMES)
+_BY_IATA = {a.iata: a for a in AERODROMES if a.iata}
+
+
+def _in_context(folded, start, end):
+    """True when an airport word sits within _WINDOW words of folded[start:end]."""
+    before = " ".join(folded[:start].split()[-_WINDOW:])
+    after = folded[end:]
+    return bool(_CONTEXT.search(before) or _CONTEXT.search(" ".join(after.split()[:_WINDOW]))
+                or _INTERNATIONAL.match(after))
+
+
+def airport_terms(query):
+    """['CYVR Vancouver', ...]: the Schedules' words for every aerodrome `query` names
+    by IATA code, a distinctive name, or a city or name beside an airport word, in
+    the order of the table; [] when it names none."""
+    found = {}
+    for token in re.findall(r"[A-Za-z]+", query or ""):
+        a = _BY_IATA.get(token.upper())
+        if a and (token.isupper() or a.iata not in _WORD_CODES):
+            found[a.icao] = a
+    folded = _fold(query)
+    covered = []    # spans a distinctive or a named match took: no city inside them
+    for a, distinct, names, _ in _ENTRIES:
+        for pattern in distinct:
+            for m in pattern.finditer(folded):
+                found[a.icao] = a
+                covered.append(m.span())
+        for pattern in names:
+            for m in pattern.finditer(folded):
+                if _in_context(folded, *m.span()):
+                    found[a.icao] = a
+                    covered.append(m.span())
+    # longest place first, so a place inside a longer one ('saint john' inside 'fort
+    # saint john') yields to it, as a city yields to a name that contains it
+    places = sorted(((m.span(), a) for a, _, _, patterns in _ENTRIES for p in patterns
+                     for m in p.finditer(folded)), key=lambda x: x[0][0] - x[0][1])
+    for (s, e), a in places:
+        if any(cs <= s and e <= ce and (cs, ce) != (s, e) for cs, ce in covered):
+            continue
+        if _in_context(folded, s, e):
+            found[a.icao] = a
+            covered.append((s, e))
+    return [f"{a.icao} {a.city}" for a in AERODROMES if a.icao in found]

canlex/amps.py

@@ -0,0 +1,157 @@
+"""Ingest the CBSA Master Penalty Document (AMPS contraventions).
+
+The Master Penalty Document is the public per-contravention reference for the
+Administrative Monetary Penalty System: each page (C001 onward) gives the
+contravention text, the penalty amounts by occurrence level, and the
+designated provision it enforces. Together with the Designated Provisions
+(Customs) Regulations schedule, it answers the trade-compliance staple
+"what does contravention X cost on a second occurrence?" -- which the corpus
+previously could not.
+
+Same fetch/cache/parse pattern as dmemo.py: index page -> per-contravention
+pages, cached under data/raw/amps, one chunk per contravention (they are
+short), doc_type='memorandum' (administrative guidance, not law).
+
+    python -m canlex.amps
+"""
+import re
+import sys
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      stored_chunks, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+INDEX_URL = "https://www.cbsa-asfc.gc.ca/trade-commerce/amps/mpd-dmi-eng.html"
+BASE = "https://www.cbsa-asfc.gc.ca"
+RAW = RAW_DIR / "amps"
+
+# Hrefs on the index may be absolute or relative; key on the c-number page
+# name alone.
+_LINK = re.compile(r'href="([^"]*?contraventions-infractions/'
+                   r'([a-z]\d{3})-eng\.html)"')
+_MODIFIED = re.compile(r'<time[^>]*>(\d{4}-\d{2}-\d{2})</time>')
+_TITLE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S)
+
+
+def _strip(html):
+    """Readable text: table cells joined with ' | ', block ends -> newlines."""
+    html = re.sub(r"</t[dh]>", " | ", html)
+    html = re.sub(r"</(?:p|li|tr|h\d|caption)>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    lines = [_norm(l).strip("| ").strip() for l in html.split("\n")]
+    return "\n".join(l for l in lines if l)
+
+
+def _main_content(html):
+    """The page's main block, shorn of nav/footer boilerplate."""
+    m = re.search(r"<main[^>]*>(.*?)</main>", html, re.S)
+    return m.group(1) if m else html
+
+
+def parse_page(html, code, url):
+    content = _main_content(html)
+    title_m = _TITLE.search(content)
+    title = _norm(re.sub(r"<[^>]+>", " ", title_m.group(1))) if title_m else code.upper()
+    body = _strip(content[title_m.end():] if title_m else content)
+    mod = _MODIFIED.search(html)
+    modified = mod.group(1) if mod else ""
+    # The first non-empty line after the title is the contravention text
+    # itself -- promote it to the marginal note so retrieval titles carry it.
+    first = next((l for l in body.split("\n") if len(l) > 30), "")
+    return {
+        "id": f"amps-{code}",
+        "doc_type": "memorandum",
+        "act_code": "AMPS",
+        "act_short": "AMPS",
+        "act_name": "AMPS Master Penalty Document",
+        "section": code.upper(),
+        "marginal_note": first[:160],
+        "part": "Master Penalty Document",
+        "division": "",
+        "heading": title,
+        "text": body,
+        "history": "",
+        "last_amended": modified,
+        "current_to": modified,
+        "citation": f"AMPS contravention {code.upper()}",
+        "source_url": url,
+    }
+
+
+def _fetch(url, dest, force=False):
+    """cbsa-asfc.gc.ca serves this section empty to Python's HTTP client;
+    route through PowerShell (like canada.ca fetches) and treat an empty
+    cache as a miss."""
+    if dest.exists() and dest.stat().st_size == 0:
+        dest.unlink()
+    data = fetch_cached(url, dest, powershell=True, pause=0.4, force=force)
+    if not data.strip():
+        raise ValueError(f"empty response for {url}")
+    return data
+
+
+def preserve_failed(chunks, failed, stored):
+    """Re-attach the last-good chunks of contraventions this run could not fetch.
+
+    Ingestion is a full rebuild, so a page that 404s or times out silently
+    leaves the corpus -- and a penalty still in force is worse missing than
+    slightly stale. `failed` carries the (code, why) pairs build() collects;
+    codes are lower-case on the index but upper-case in a chunk's `section`,
+    so match on the upper-cased form. Restricted to codes that actually
+    errored, so a contravention CBSA has genuinely retired stays gone.
+    Returns (chunks, preserved codes).
+    """
+    codes = {code.upper() for code, _why in failed}
+    return preserve_dropped(chunks, stored, key=lambda c: c["section"].upper(),
+                            only=codes)
+
+
+def build(limit=None, allow_shrink=False, force=False):
+    html = _fetch(INDEX_URL, RAW / "index.html",
+                  force=force).decode("utf-8", "replace")
+    links = sorted({(m.group(1), m.group(2)) for m in _LINK.finditer(html)},
+                   key=lambda t: t[1])
+    if limit:
+        links = links[:limit]
+    print(f"amps: {len(links)} contravention pages on the index")
+    chunks, failed = [], []
+    for path, code in links:
+        if path.startswith("http"):
+            url = path
+        elif path.startswith("/"):
+            url = BASE + path
+        else:
+            url = INDEX_URL.rsplit("/", 1)[0] + "/" + path
+        try:
+            page = _fetch(url, RAW / f"{code}.html",
+                          force=force).decode("utf-8", "replace")
+            chunks.append(parse_page(page, code, url))
+        except Exception as exc:
+            failed.append((code, f"{type(exc).__name__}: {exc}"))
+    out = PROCESSED_DIR / "amps.json"
+    # Printed before the write so a refusal still shows what went wrong.
+    if failed:
+        print(f"amps: {len(failed)} failed: {failed[:5]}")
+    if not limit:
+        chunks, preserved = preserve_failed(chunks, failed, stored_chunks(out))
+        if preserved:
+            print(f"amps: kept the last-good copy of {len(preserved)} "
+                  f"unfetchable contravention(s): {', '.join(preserved)}")
+    # The guard applies to --limit runs too: a smoke test over five
+    # contraventions must not be able to write a five-page corpus over the real
+    # one. indent=2 matches what is already on disk -- changing it would
+    # rewrite every line of the file.
+    if not write_corpus(out, chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"amps: {len(chunks)} contraventions -> {out.name}")
+    return True
+
+
+def main():
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/authorities.py

@@ -0,0 +1,541 @@
+"""The authority registry (roadmap T3.2; track A step A12, 2026-09-14).
+
+data/curated/authorities.json (HELD FOR REVIEW) links every decision the
+curated datasets and the code cite to the corpus: a stable key per authority,
+an alias table of the exact cite strings used, the caselaw.json decision key
+(act_short / act_code, exact), pinned chunks each with a verbatim 'contains'
+phrase, an ordered list of the secondary chunks that describe an authority
+the corpus does not hold (official guidance before archived IRB papers,
+before withdrawn ATIP copies, before guidance that cites the reversed trial
+decision in Saini), a status (in-corpus / secondary-only / unavailable) and
+the subsequent history. resolve() reads a cite and returns the pinned chunk
+and its excerpt; the pointer lines tell an officer where the text is:
+
+    full text in CanLex: canlex_search_legislation doc_type caselaw, act=...
+    not in the CanLex corpus; quoted in <secondary>
+
+Never canlex_case -- it needs a CanLII key and returns the citator, not the
+decision. The file is optional: a composed release ships without held files,
+and every renderer then prints HELD_NOTE instead of pointers.
+
+Rendering is gated on the registry's own top-level "render": true (2026-09-15,
+wave-3 verifier). The approved f5cf54d registry has no such flag: its notes
+record corrections to the curated holdings that were never applied, so a
+release composed on that baseline prints no pointers (and keeps
+server.SUPERSEDED_CITES) rather than a 'full text in CanLex' pointer beside an
+uncorrected holding. The corrected registry sets the flag.
+
+verify_lines() prints, wherever a tool shows a pairing's verified maximum, the
+sentence of the statute that contains each verify phrase, tagged with the Act,
+section, current_to and chunk id -- read from data/processed, never typed.
+Where the phrase occurs more than once (s. 344's 'imprisonment for life' is in
+the firearm clause (a) and in '(b) in any other case'), the general clause is
+quoted.
+"""
+import functools
+import json
+import re
+
+from .config import DATA_DIR, PROCESSED_DIR
+
+FILE = "authorities.json"
+PATH = DATA_DIR / "curated" / FILE
+
+STATUSES = ("in-corpus", "secondary-only", "unavailable")
+# The order secondary chunks are listed in (official first; the reversed
+# trial decision's citers last).
+SECONDARY_ORDER = ("in-corpus decision", "official", "archived IRB paper",
+                   "withdrawn (unofficial ATIP copy)", "cites trial-level Saini (reversed)")
+
+# ---- officer-facing wording (registered in canlex/reviewed_code.py) ----------
+HELD_NOTE = ("Authority registry (data/curated/authorities.json): HELD FOR REVIEW -- not "
+             "in this deployment, so the cites above are not linked to the corpus. Where a "
+             "decision is in the corpus, canlex_search_legislation (doc_type 'caselaw') "
+             "finds it by name; canlex_case needs a CanLII key and returns the citator, not "
+             "the decision.")
+READ_NOTE = ("Each authority above says where its text is: 'full text in CanLex' means "
+             "canlex_search_legislation (doc_type 'caselaw', act=<name>) returns the "
+             "decision; 'not in the CanLex corpus' means only the listed guidance or later "
+             "decisions describe it. The registry's subsequent-history line is the only "
+             "treatment check CanLex holds -- confirm a decision's currency elsewhere "
+             "before relying on it.")
+# The tail printed when this deployment's registry is not rendered (the
+# approved f5cf54d baseline): no pointers above, and still never canlex_case.
+NO_POINTER_NOTE = ("Retrieve a cited decision's text with canlex_search_legislation "
+                   "(doc_type 'caselaw', the decision's name) where the CanLex corpus "
+                   "holds it. CanLex holds no treatment check beyond the SUBSEQUENT "
+                   "HISTORY notes it prints -- confirm a decision's currency elsewhere "
+                   "before relying on it.")
+UNLINKED = "not in the CanLex authority registry (unlinked cite; see data/curated/authorities.json)"
+STATUS_TEXT = {
+    "in-corpus": "full text in CanLex: canlex_search_legislation doc_type caselaw, act={short!r}",
+    "absent": ("not in this deployment's corpus (the registry lists it as in-corpus, "
+               "act={short!r}); quoted in {where}"),
+    "secondary-only": "not in the CanLex corpus; quoted in {where}",
+    "unavailable": "not in the CanLex corpus ({why}); described in {where}",
+}
+NO_SECONDARY = "no corpus source describes it"
+VERIFY_LABEL = "VERIFIED TEXT"
+VERIFY_MISSING = ("{act} s. {section}: corpus text not available in this deployment "
+                  "(phrase: {phrase!r})")
+VERIFY_NOT_FOUND = ("{act} s. {section}{when}: phrase {phrase!r} NOT FOUND in the corpus "
+                    "text -- the quoted maximum needs re-verification")
+
+_QUOTES = str.maketrans({"“": '"', "”": '"', "‘": "'", "’": "'",
+                         " ": " "})
+
+
+def _norm(s):
+    return re.sub(r"\s+", " ", (s or "").translate(_QUOTES)).strip()
+
+
+def _fold(cite):
+    """A cite string as compared: case, spacing and dotted initials folded, so
+    'Canada (M.C.I.) v Saini' still finds 'Canada (MCI) v Saini'."""
+    s = _norm(cite).lower().replace(".", "")
+    return re.sub(r"\s+", " ", s)
+
+
+# ---- the registry ---------------------------------------------------------------
+
+@functools.lru_cache(maxsize=4)
+def _load(path_str):
+    try:
+        with open(path_str, encoding="utf-8") as fh:
+            data = json.loads(fh.read())
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict) or not isinstance(data.get("authorities"), dict):
+        return None
+    folded = {_fold(cite): key for cite, key in (data.get("aliases") or {}).items()}
+    data["_folded"] = folded
+    return data
+
+
+def load(path=None):
+    """The registry, or None when this deployment lacks the (held) file."""
+    return _load(str(path or PATH))
+
+
+def renderable(registry=None):
+    """True when the registry says its rows may be printed beside the curated
+    holdings (top-level "render": true -- see the module docstring)."""
+    reg = registry if registry is not None else load()
+    return bool(reg and reg.get("render") is True)
+
+
+def read_note(registry=None):
+    """The closing line of an answer that cites decisions: READ_NOTE when the
+    registry's pointers are printed, NO_POINTER_NOTE otherwise."""
+    return READ_NOTE if renderable(registry) else NO_POINTER_NOTE
+
+
+def reload():
+    _load.cache_clear()
+    _corpus_file.cache_clear()
+    _act_sections.cache_clear()
+    statute_sentence.cache_clear()
+
+
+def lookup(cite, registry=None):
+    """(key, row) for an exact cite string (folded), or (None, None)."""
+    reg = registry if registry is not None else load()
+    if not reg or not cite:
+        return None, None
+    key = (reg.get("aliases") or {}).get(cite) or reg["_folded"].get(_fold(cite))
+    if key is None and cite in reg["authorities"]:
+        key = cite
+    if key is None:
+        return None, None
+    return key, reg["authorities"].get(key)
+
+
+# ---- the corpus: caselaw keys and the chunks the registry names ------------------
+
+def _file_for(chunk_id):
+    if chunk_id.startswith("enf-fr-"):
+        return "enf-fr.json"
+    if chunk_id.startswith("enf-"):
+        return "enf.json"
+    if chunk_id.startswith("irb-ls-"):
+        return "irb_papers.json"
+    if chunk_id.startswith("irb-"):
+        return "irb_guidelines.json"
+    if chunk_id.startswith("pdi-"):
+        return "pdi.json"
+    if chunk_id.startswith("commentary-"):
+        return "commentary.json"
+    if chunk_id.startswith("policy-"):
+        return "policy.json"
+    return "caselaw.json"
+
+
+def _wanted_ids(registry):
+    ids = set()
+    for row in (registry or {}).get("authorities", {}).values():
+        ids.update(p["chunk"] for p in row.get("pinned") or [])
+        ids.update(s["chunk"] for s in row.get("secondary") or [])
+    return frozenset(ids)
+
+
+@functools.lru_cache(maxsize=8)
+def _corpus_file(name, wanted):
+    """(decision keys, citations, {id: chunk}) of one processed file, keeping
+    only the chunks the registry names (caselaw.json is 20 MB)."""
+    keys, citations, chunks = set(), set(), {}
+    try:
+        data = json.loads((PROCESSED_DIR / name).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    for c in data:
+        if c.get("doc_type") == "caselaw":
+            keys.add((c.get("act_short"), c.get("act_code")))
+            citations.add(c.get("citation") or "")
+        if c.get("id") in wanted:
+            chunks[c["id"]] = c
+    return keys, citations, chunks
+
+
+def _corpus(name, registry=None):
+    reg = registry if registry is not None else load()
+    return _corpus_file(name, _wanted_ids(reg))
+
+
+def chunk(chunk_id, registry=None):
+    """A chunk the registry names, from the processed corpus, or None."""
+    got = _corpus(_file_for(chunk_id), registry)
+    return (got[2].get(chunk_id) if got else None)
+
+
+def decision_present(decision_key, index_chunks=None):
+    """True / False when the corpus does / does not hold the decision the key
+    names (the loaded index's chunks when given, else data/processed);
+    None when neither is readable."""
+    if not decision_key:
+        return False
+    want = (decision_key.get("act_short"), decision_key.get("act_code"))
+    if index_chunks is not None:
+        return want in _decision_keys(index_chunks)
+    got = _corpus("caselaw.json")
+    if got is None:
+        return None
+    return want in got[0]
+
+
+_KEYS_SEEN = {}
+
+
+def _decision_keys(index_chunks):
+    """{(act_short, act_code)} of the caselaw chunks of a loaded index,
+    computed once per chunk list (a tool answer asks for every authority)."""
+    got = _KEYS_SEEN.get(id(index_chunks))
+    if got is None or got[0] != len(index_chunks):
+        keys = {(c.get("act_short"), c.get("act_code")) for c in index_chunks
+                if c.get("doc_type") == "caselaw"}
+        _KEYS_SEEN.clear()
+        _KEYS_SEEN[id(index_chunks)] = got = (len(index_chunks), keys)
+    return got[1]
+
+
+def excerpt(chunk_id, phrase, registry=None, width=280):
+    """The sentence of the chunk that contains the phrase (the phrase itself
+    when the chunk is not readable here)."""
+    c = chunk(chunk_id, registry)
+    if c is None:
+        return None
+    return _sentence(c.get("text") or "", phrase, width)
+
+
+# A period that ends one of these does not end a sentence ('Mr. Drake',
+# 'para. 24', 'J. dissenting', 's. 36', 'No. 363').
+_ABBREV = re.compile(r"(?:\b(?:Mr|Mrs|Ms|Dr|Hon|J|JJ|JA|No|Nos|v|vs|para|paras|s|ss|subs|p|pp|"
+                     r"art|arts|c|ch|cf|e\.g|i\.e|R\.S\.C|S\.C|F\.C|D\.L\.R|S\.C\.R|F\.T\.R|"
+                     r"Imm\. L\.R|N\.R|F\.C\.J)|\b[A-Z])$")
+_BOUNDARY = re.compile(r"[.;:](?=\s+[A-Z(\[\"]|\s*$)")
+
+
+def _boundaries(t):
+    """Offsets just after each sentence end in t (abbreviations skipped)."""
+    out = []
+    for m in _BOUNDARY.finditer(t):
+        if m.group(0) == "." and _ABBREV.search(t[:m.start()]):
+            continue
+        out.append(m.end())
+    return out
+
+
+# The general limb of a punishment clause: where a maximum phrase occurs more
+# than once in a section ('(a) if a restricted firearm ... is used ..., to
+# imprisonment for life ...; and (b) in any other case, to imprisonment for
+# life', ss. 273 and 344), the occurrence after this formula is the one that
+# states the maximum for every case (wave-3 verifier, 2026-09-15).
+_GENERAL_CLAUSE = re.compile(r"in any other case,?\s+(?:to\s+|by\s+)?$", re.I)
+
+
+_CLAUSE_MARK = re.compile(r"\([a-z](?:\.\d+)?\)\s*in any other case", re.I)
+
+
+def _general_clause_quote(t, first, i, n):
+    """'<the punishment sentence's lead-in> ... (b) in any other case, to
+    <maximum>': the words that make the clause a maximum, and the clause,
+    without the special cases between them (each part verbatim)."""
+    ends = _boundaries(t)
+    s0 = max([e for e in ends if e <= first], default=0)
+    mark = re.search(r"\(a\)", t[s0:first])
+    clause = list(_CLAUSE_MARK.finditer(t, max(0, i - 80), i))
+    if not mark or not clause:
+        return None
+    end = next((e for e in ends if e >= i + n), len(t))
+    return f"{t[s0:s0 + mark.start()].strip()} ... {t[clause[-1].start():end].strip()}"
+
+
+def _sentence(text, phrase, width=280, general=False):
+    t = _norm(text)
+    p = _norm(phrase)
+    low, pl = t.lower(), p.lower()
+    hits = [m.start() for m in re.finditer(re.escape(pl), low)] if pl else []
+    if not hits:
+        return None
+    i = hits[0]
+    if general:
+        i = next((h for h in hits if _GENERAL_CLAUSE.search(t[max(0, h - 40):h])), i)
+        if i != hits[0]:
+            lead = _general_clause_quote(t, hits[0], i, len(p))
+            if lead:
+                return lead
+    ends = _boundaries(t)
+    before = [e for e in ends if e <= i]
+    start = before[-1] if before else 0
+    after = [e for e in ends if e >= i + len(p)]
+    end = after[0] if after else len(t)
+    s = t[start:end].strip()
+    if len(s) > 2 * width + len(p):
+        a, b = max(start, i - width), min(end, i + len(p) + width)
+        s = ("..." if a > start else "") + t[a:b].strip() + ("..." if b < end else "")
+    return s
+
+
+# ---- resolve --------------------------------------------------------------------
+
+def resolve(cite, registry=None, index_chunks=None, with_excerpts=True):
+    """What the registry knows about a cite: key, name, status, whether this
+    corpus holds the decision, the pinned chunks with their excerpts, the
+    ordered secondary chunks, the subsequent history. None when the registry
+    is absent; {'unlinked': True} when the cite is not in it."""
+    reg = registry if registry is not None else load()
+    if reg is None:
+        return None
+    key, row = lookup(cite, reg)
+    if row is None:
+        return {"cite": cite, "unlinked": True}
+    present = (decision_present(row.get("decision_key"), index_chunks)
+               if row.get("status") == "in-corpus" else False)
+    pinned = []
+    for p in row.get("pinned") or []:
+        e = dict(p)
+        if with_excerpts:
+            c = chunk(p["chunk"], reg)
+            e["excerpt"] = _sentence(c.get("text") or "", p["contains"]) if c else None
+            e["marginal_note"] = (c or {}).get("marginal_note")
+        pinned.append(e)
+    return {"cite": cite, "key": key, "name": row.get("name"), "status": row.get("status"),
+            "present": present, "decision_key": row.get("decision_key"),
+            "citation": row.get("citation"), "court": row.get("court"),
+            "decided": row.get("decided"), "pinned": pinned,
+            "secondary": list(row.get("secondary") or []),
+            "subsequent_history": row.get("subsequent_history"),
+            "unavailable_reason": row.get("unavailable_reason"),
+            "parallel_cites": list(row.get("parallel_cites") or []),
+            "notes": list(row.get("notes") or [])}
+
+
+def _where(res, limit=4):
+    secs = res.get("secondary") or []
+    if not secs:
+        return NO_SECONDARY
+    parts = []
+    for s in secs[:limit]:
+        dated = f", {s['current_to']}" if s.get("current_to") else ""
+        parts.append(f"{s['label']} [{s['chunk']}; {s['status']}{dated}]")
+    more = len(secs) - limit
+    return "; ".join(parts) + (f"; and {more} more" if more > 0 else "")
+
+
+def _officer_pin(p):
+    """A pin's label as an officer may read it, or None. Reviewer notes live
+    in a pin's 'review_note' (never printed); a label that still talks about
+    the curated data ('the curated (para 11) is wrong' -- the approved
+    registry's style) is a reviewer note too, and is not printed."""
+    label = (p or {}).get("pin") or ""
+    return None if (not label or re.search(r"curated|->", label)) else label
+
+
+def _pick_pin(res, pin):
+    """The registry pin that matches the curated pin's first number, else the
+    first pinned chunk that is not a heading-only pin."""
+    pins = res.get("pinned") or []
+    if not pins:
+        return None
+    nums = re.findall(r"\d+", pin or "")
+    if nums:
+        for p in pins:
+            if re.search(rf"\b{nums[0]}\b", p.get("pin") or "") and not p.get("note"):
+                return p
+    return next((p for p in pins if not p.get("note")), pins[0])
+
+
+def pointer(res, pin=None, brief=False):
+    """The officer-facing line under a cited authority (resolve()'s result)."""
+    if res is None:
+        return HELD_NOTE
+    if res.get("unlinked"):
+        return UNLINKED
+    short = (res.get("decision_key") or {}).get("act_short")
+    status = res["status"]
+    if status == "in-corpus" and res.get("present") is not False:
+        line = STATUS_TEXT["in-corpus"].format(short=short)
+        if res.get("present") is None:
+            line += " (presence in this deployment not checked)"
+        p = _pick_pin(res, pin)
+        if p:
+            label = _officer_pin(p)
+            line += (f"; pinned {label}, chunk {p['chunk']}" if label
+                     else f"; pinned chunk {p['chunk']}")
+            if not brief and p.get("excerpt"):
+                line += f': "{p["excerpt"]}"'
+        if brief:
+            return line
+        if res.get("subsequent_history") and res["subsequent_history"] != "None found.":
+            line += f"\n  subsequent history: {res['subsequent_history']}"
+        return line
+    where = _where(res)
+    if status == "in-corpus":
+        line = STATUS_TEXT["absent"].format(short=short, where=where)
+    elif status == "unavailable":
+        why = res.get("unavailable_reason") or "unavailable"
+        line = (STATUS_TEXT["unavailable"].format(why=why, where=where)
+                if res.get("secondary") else f"not in the CanLex corpus ({why}); {NO_SECONDARY}")
+    else:
+        line = STATUS_TEXT["secondary-only"].format(where=where)
+    if not brief and res.get("subsequent_history") and res["subsequent_history"] != "None found.":
+        line += f"\n  subsequent history: {res['subsequent_history']}"
+    return line
+
+
+def authority_lines(auths, registry=None, brief=False, index_chunks=None):
+    """The 'Authorities:' bullets of an entry or pairing, each followed by its
+    registry pointer (or HELD_NOTE once when the registry is absent; the bare
+    bullets when the registry is not rendered -- see renderable())."""
+    reg = registry if registry is not None else load()
+    lines = []
+    show = reg is not None and renderable(reg)
+    for a in auths or []:
+        pin = f", {a['pin']}" if a.get("pin") else ""
+        lines.append(f"- {a['cite']} ({a.get('court', '')}{pin}): {a.get('holding', '')}")
+        if not show:
+            continue
+        res = resolve(a["cite"], reg, index_chunks=index_chunks, with_excerpts=not brief)
+        text = pointer(res, a.get("pin"), brief=brief)
+        lines += ["  " + t for t in text.split("\n")]
+    if auths and reg is None:
+        lines.append(f"  ({HELD_NOTE})" if brief else f"  {HELD_NOTE}")
+    return lines
+
+
+# ---- subsequent history: the reversed decisions guidance still cites -------------
+
+def superseded_entries(registry=None):
+    """SUPERSEDED_CITES-shaped entries from the registry rows that carry a
+    'superseded' block (the reversed trial decision in Saini), or [] when the
+    registry is absent or not rendered -- canlex/server.py then uses its own
+    constant."""
+    reg = registry if registry is not None else load()
+    if not renderable(reg):
+        return []
+    out = []
+    for key, row in sorted((reg or {}).get("authorities", {}).items()):
+        s = row.get("superseded")
+        if not s:
+            continue
+        try:
+            pattern = re.compile(s["pattern"], re.S)
+        except (re.error, KeyError):
+            continue
+        out.append({"id": key, "pattern": pattern, "note": s.get("note", ""),
+                    "appellate_citation": s.get("appellate_citation", ""),
+                    "read_present": s.get("read_present", ""),
+                    "read_absent": s.get("read_absent", ""),
+                    "read_unknown": s.get("read_unknown", "")})
+    return out
+
+
+# ---- the statute sentence behind each verified maximum --------------------------
+
+@functools.lru_cache(maxsize=16)
+def _act_sections(act):
+    """{section: [chunk]} of an Act's English chunks, or None when the
+    processed file is absent here."""
+    try:
+        data = json.loads((PROCESSED_DIR / f"{act}.json").read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    out = {}
+    for c in data:
+        if c.get("lang", "en") != "en":
+            continue
+        out.setdefault(str(c.get("section")), []).append(c)
+    return out
+
+
+@functools.lru_cache(maxsize=512)
+def statute_sentence(act, section, phrase):
+    """{act, section, current_to, chunk, sentence} for the corpus sentence
+    that contains a verify phrase; {'missing': True} when the file is absent;
+    {'not_found': True, 'current_to': ...} when the phrase is gone (a corpus
+    refresh changed the text)."""
+    sections = _act_sections(act)
+    if sections is None:
+        return {"act": act, "section": section, "phrase": phrase, "missing": True}
+    chunks = sections.get(str(section)) or []
+    for c in chunks:
+        s = _sentence(c.get("text") or "", phrase, general=True)
+        if s:
+            return {"act": act, "section": section, "phrase": phrase,
+                    "current_to": c.get("current_to") or "", "chunk": c.get("id"),
+                    "sentence": s}
+    return {"act": act, "section": section, "phrase": phrase, "not_found": True,
+            "current_to": (chunks[0].get("current_to") if chunks else "") or ""}
+
+
+def verify_rows(pairing, limb=None):
+    """The pairing's verify rows plus the selected limb's, without repeats."""
+    rows, seen = [], set()
+    for v in list(pairing.get("verify") or []) + (
+            [v for l in pairing.get("limbs") or [] if l.get("key") == limb
+             for v in l.get("verify") or []] if limb else []):
+        key = (v.get("act"), v.get("section"), v.get("contains"))
+        if key in seen or not all(key):
+            continue
+        seen.add(key)
+        rows.append(v)
+    return rows
+
+
+def verify_lines(pairing, limb=None):
+    """The VERIFIED TEXT lines printed under a pairing's maximum: for each
+    verify phrase, the statute sentence that contains it, with the Act, the
+    section, the corpus 'current to' date and the chunk id."""
+    lines = []
+    for v in verify_rows(pairing, limb):
+        r = statute_sentence(v["act"], str(v["section"]), v["contains"])
+        if r.get("missing"):
+            lines.append(f"{VERIFY_LABEL}: " + VERIFY_MISSING.format(
+                act=v["act"], section=v["section"], phrase=v["contains"]))
+        elif r.get("not_found"):
+            when = f" (current to {r['current_to']})" if r.get("current_to") else ""
+            lines.append(f"{VERIFY_LABEL}: " + VERIFY_NOT_FOUND.format(
+                act=v["act"], section=v["section"], when=when, phrase=v["contains"]))
+        else:
+            lines.append(f"{VERIFY_LABEL}: {r['act']} s. {r['section']} (current to "
+                         f"{r['current_to'] or 'n/a'}, chunk {r['chunk']}): \"{r['sentence']}\"")
+    return lines

canlex/benefits.py

@@ -0,0 +1,352 @@
+"""Ingest the member booklets of the federal public-service benefit plans.
+
+Four plans cover federal employees (and a Border Services Officer over a
+career), by population:
+
+  - PSHCP -- Public Service Health Care Plan (health; active + pensioners)
+  - PSDCP -- Public Service Dental Care Plan (dental; active employees)
+  - PDSP  -- Pensioners' Dental Services Plan (dental; pensioners)
+  - PSMIP -- Public Service Management Insurance Plan (life / AD&D / disability
+             insurance; management-category and excluded employees -- so EX and
+             excluded CBSA staff rather than the represented FB group, who carry
+             the separate Disability Insurance Plan)
+
+Each plan's *member booklet* is the plain-language guide the plan administrator
+or Treasury Board publishes -- what is covered, who is eligible, how to claim,
+the appeals process. The booklets are guidance: the governing terms live in the
+PSHCP Directive (already ingested as doc_type='directive') and the respective
+plan rules, which prevail in any discrepancy. They are tagged doc_type=
+'benefits' and labelled as such in results.
+
+Sources: the Canada Life PDFs (welcome.canadalife.com / canadalife.com), which
+-- unlike canada.ca -- serve Python's urllib fine; and the PSMIP booklet, an
+HTML page on canada.ca (which blocks urllib at the TLS layer, so it is fetched
+via PowerShell, as agreement.py does). A PDF booklet carries a table of
+contents whose headings are located in the body to segment it; the HTML booklet
+is segmented at its h2/h3 headings. Either way a retrieved chunk is a coherent
+benefit topic.
+
+    py -m canlex.benefits
+"""
+import io
+import re
+import sys
+
+from bs4 import BeautifulSoup
+from pypdf import PdfReader
+
+from ._common import (dehyphenate, document_words, fetch_cached, norm_ws,
+                      preserve_dropped, split_lines, stored_chunks,
+                      write_corpus)
+
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "benefits"
+OUT = PROCESSED_DIR / "benefits.json"
+
+_CHUNK_CHARS = 1800     # target characters per chunk
+
+# Each plan: its booklet, and how to read it. PDF booklets give the page
+# indices of the table of contents and the first body page (everything before
+# it is cover + TOC). The HTML booklet is parsed at its h2/h3 headings.
+# 'edition' is the booklet's own date (for HTML it is read from the page's
+# "Date modified"); bump a PDF edition and re-run when a new booklet is posted.
+SOURCES = {
+    "PSHCP": {
+        "code": "PSHCP", "format": "pdf",
+        "name": "Public Service Health Care Plan",
+        "url": ("https://www.welcome.canadalife.com/content/dam/rfp/"
+                "welcome-sites/pshcp/PSHCP-member-booklet.pdf"),
+        "label": "Member Booklet",
+        "toc_pages": [2, 3, 4],
+        "body_start": 5,
+        "edition": "June 2024",
+    },
+    "PSDCP": {
+        "code": "PSDCP", "format": "pdf",
+        "name": "Public Service Dental Care Plan",
+        "url": ("https://www.welcome.canadalife.com/content/dam/rfp/"
+                "welcome-sites/psdcp/psdcp-member-booklet.pdf"),
+        "label": "Member Booklet",
+        "toc_pages": [2, 3],
+        "body_start": 4,
+        "edition": "May 2026",
+    },
+    "PDSP": {
+        "code": "PDSP", "format": "pdf",
+        "name": "Pensioners' Dental Services Plan",
+        "url": ("https://www.canadalife.com/content/dam/rfp/welcome-sites/"
+                "pdsp/pdsp-member-booklet.pdf"),
+        "label": "Member Booklet",
+        "toc_pages": [2, 3],
+        "body_start": 4,
+        "edition": "November 2024",
+    },
+    "PSMIP": {
+        "code": "PSMIP", "format": "html",
+        "name": "Public Service Management Insurance Plan",
+        "url": ("https://www.canada.ca/en/treasury-board-secretariat/services/"
+                "benefit-plans/management-insurance-plan/main-plan-booklet.html"),
+        "label": "Main Plan Booklet",
+        # The canada.ca page exposes no machine-readable date (its <time> tags are
+        # content dates, not a page revision). Rather than render currency as
+        # 'n/a', state plainly that the source is undated so a reader knows to
+        # verify rather than assume staleness.
+        "edition": "undated (TBS web edition)",
+    },
+}
+
+# canada.ca section labels that are navigation/footer, not plan content.
+_SKIP_HEADINGS = {"on this page", "page details", "from:", "table of contents"}
+
+# A table-of-contents line: optional dot-leaders, the heading text, then the
+# printed page number. The heading must end in a letter or ')' so a bare page
+# number or a dot run alone is not mistaken for a heading.
+_TOC_LINE = re.compile(r"^[\s.]*(.*?[A-Za-z)])\s*\.*\s*(\d{1,3})\s*$")
+
+
+def _fetch(url, dest, force=False):
+    return fetch_cached(url, dest, timeout=90, pause=0, force=force)
+
+
+def _clean(text):
+    """Tidy extracted PDF text: turn bullets into dashes, collapse runs of
+    spaces, and trim excess blank lines. Apostrophes, accents and en-dashes
+    extract as proper Unicode, so no transliteration is needed."""
+    text = text.replace("•", "\n- ").replace("\xa0", " ")
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n[ \t]+", "\n", text)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _page_texts(pdf_bytes):
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    return [p.extract_text() or "" for p in reader.pages]
+
+
+def parse_toc(page_texts, toc_pages):
+    """Return the ordered, de-duplicated list of section headings from the
+    table of contents."""
+    seen, headings = set(), []
+    for i in toc_pages:
+        if i >= len(page_texts):
+            continue
+        for line in page_texts[i].split("\n"):
+            line = line.strip()
+            if not line or "Table of Contents" in line:
+                continue
+            m = _TOC_LINE.match(line)
+            if not m:
+                continue
+            title = re.sub(r"\s+", " ", m.group(1)).strip(" .")
+            if len(title) > 2 and title.lower() not in seen:
+                seen.add(title.lower())
+                headings.append(title)
+    return headings
+
+
+def segment(body, headings):
+    """Split the body text at each heading, in order, returning a list of
+    (heading, section_text). A heading the extractor split across lines is
+    matched with flexible whitespace; a heading not found in the body is
+    skipped (its text folds into the preceding section)."""
+    cuts = []
+    pos = 0
+    for h in headings:
+        pattern = re.compile(r"\s*".join(re.escape(w) for w in h.split()), re.I)
+        m = pattern.search(body, pos)
+        if m:
+            cuts.append((h, m.start()))
+            pos = m.end()
+    sections = []
+    for j, (h, start) in enumerate(cuts):
+        end = cuts[j + 1][1] if j + 1 < len(cuts) else len(body)
+        sections.append((h, body[start:end].strip()))
+    return sections
+
+
+def _split(text, target=_CHUNK_CHARS):
+    """Split a long section into ~target-sized pieces at line boundaries,
+    folding small leading/trailing remnants into a neighbour so a section
+    never yields a stub chunk."""
+    return split_lines(text, target, fold_stubs=True)
+
+
+def _slug(heading, used):
+    """A short, unique, citable section locator from a heading."""
+    base = re.sub(r"[^a-z0-9]+", "-", heading.lower()).strip("-")[:40] or "section"
+    slug, n = base, 2
+    while slug in used:
+        slug = f"{base}-{n}"
+        n += 1
+    used.add(slug)
+    return slug
+
+
+def _emit(sections, src):
+    """Build CanLex chunk dicts from a list of (heading, section_text), shared
+    by the PDF and HTML paths. Skips pure section-divider headings and
+    sub-splits any section over the chunk budget."""
+    chunks = []
+    used_slugs = set()
+    label = src["label"]
+    for heading, text in sections:
+        # Skip pure section dividers -- a heading whose only content is the
+        # heading itself (its substance lives in the child sections below it).
+        # Compare with whitespace collapsed, since a PDF extractor often breaks
+        # a heading across lines ("...Plan \n(PSDCP)").
+        norm_text = norm_ws(text)
+        norm_head = norm_ws(heading)
+        remainder = (norm_text[len(norm_head):]
+                     if norm_text.lower().startswith(norm_head.lower())
+                     else norm_text)
+        if len(remainder.strip()) < 40:
+            continue
+        slug = _slug(heading, used_slugs)
+        pieces = _split(text)
+        for k, piece in enumerate(pieces, start=1):
+            section = slug if len(pieces) == 1 else f"{slug}-{k}"
+            note = heading if len(pieces) == 1 else f"{heading} (part {k})"
+            chunks.append({
+                "id": f"benefits-{src['code']}-{section}",
+                "doc_type": "benefits",
+                "act_code": src["code"],
+                "act_short": src["code"],
+                "act_name": f"{src['name']} — {label}",
+                "section": section,
+                "marginal_note": note,
+                "part": f"{src['name']} {label}",
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": "",
+                "current_to": src["edition"],
+                "citation": (f"{src['code']} {label}"
+                             + (f" ({src['edition']})" if src["edition"] else "")
+                             + f", “{heading}”"),
+                "source_url": src["url"],
+            })
+    return chunks
+
+
+def parse_pdf_booklet(pdf_bytes, src):
+    """Segment a PDF booklet by its table-of-contents headings."""
+    pages = _page_texts(pdf_bytes)
+    headings = parse_toc(pages, src["toc_pages"])
+    body = _clean("\n".join(pages[src["body_start"]:]))
+    # line-break hyphens closed up (keeping the hyphen unless the booklet
+    # itself uses the joined word; 2026-09-22)
+    body = dehyphenate(body, document_words(pages))
+    return _emit(segment(body, headings), src)
+
+
+def _html_block_text(heading):
+    """Readable text from an h2/h3 up to the next heading of the same or higher
+    level; lists become dash items."""
+    lines = []
+    for sib in heading.find_next_siblings():
+        if sib.name in ("h2", "h3"):
+            break
+        if sib.name in ("ul", "ol"):
+            for li in sib.find_all("li", recursive=False):
+                item = norm_ws(li.get_text(" ", strip=True))
+                if item:
+                    lines.append(f"- {item}")
+        elif sib.name in ("p", "div", "table", "section", "h4", "h5"):
+            text = norm_ws(sib.get_text(" ", strip=True))
+            if text:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def parse_html_booklet(html, src):
+    """Segment the canada.ca HTML booklet at its h2/h3 headings."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main") or soup
+    # Read the page's "Date modified" so the booklet's currency is recorded.
+    if not src["edition"]:
+        t = main.find("time", property="dateModified") or soup.find(
+            "time", property="dateModified")
+        if t:
+            src = {**src, "edition": t.get_text(strip=True)}
+    sections = []
+    for h in main.find_all(["h2", "h3"]):
+        title = norm_ws(h.get_text(" ", strip=True))
+        if not title or title.lower() in _SKIP_HEADINGS:
+            continue
+        body = _html_block_text(h)
+        if body:
+            sections.append((title, f"{title}\n{body}"))
+    return _emit(sections, src)
+
+
+def _fetch_html(url, dest, force=False):
+    """Fetch a canada.ca page via PowerShell -- the site blocks Python's HTTP
+    client at the TLS layer, but accepts PowerShell's .NET stack."""
+    return fetch_cached(url, dest, powershell=True, force=force)
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of plans this run could not rebuild.
+
+    Ingestion is a full rebuild of all four booklets into one file, so one plan
+    failing -- Canada Life moves a PDF, canada.ca times out, a re-typeset
+    booklet no longer segments -- silently deletes that plan's coverage while
+    the other three look healthy. A plan's chunks all carry its code in
+    act_code, so the stored copy can be restored intact. Restricted to plans
+    that actually failed, so a plan removed from SOURCES on purpose (a wound-up
+    plan) is not resurrected. Returns (chunks, preserved plan codes).
+    """
+    failed = {code for code, _why in failures}
+    return preserve_dropped(chunks, stored, key=lambda c: c["act_code"],
+                            only=failed)
+
+
+def build(allow_shrink=False, force=False):
+    all_chunks, failures = [], []
+    for src in SOURCES.values():
+        print(f"Ingesting {src['code']} {src['label'].lower()} ...")
+        try:
+            if src["format"] == "pdf":
+                pdf = _fetch(src["url"], RAW / f"{src['code']}.pdf",
+                             force=force)
+                chunks = parse_pdf_booklet(pdf, src)
+            else:
+                html = _fetch_html(src["url"], RAW / f"{src['code']}.html",
+                                   force=force).decode("utf-8", "replace")
+                chunks = parse_html_booklet(html, src)
+        except Exception as exc:
+            print(f"  !! {src['code']}: {type(exc).__name__}: {exc}")
+            failures.append((src["code"], f"{type(exc).__name__}: {exc}"))
+            continue
+        if not chunks:
+            # A booklet that downloads fine but segments to nothing has failed
+            # just as surely as one that 404s -- that is how the CBSA index
+            # loss looked -- so it counts as a failure rather than a plan that
+            # legitimately has no content.
+            print(f"  !! {src['code']}: no sections parsed")
+            failures.append((src["code"], "no sections parsed"))
+            continue
+        all_chunks.extend(chunks)
+        print(f"  {len(chunks)} chunks")
+
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} booklet(s): "
+              f"{', '.join(preserved)}")
+    if not write_corpus(OUT, all_chunks, indent=1, allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} benefit-plan chunks from {len(SOURCES)} "
+          f"booklet(s) -> {OUT}")
+    return True
+
+
+def main():
+    sys.exit(0 if build(allow_shrink="--allow-shrink" in sys.argv,
+                        force="--force" in sys.argv) else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/bm25.py

@@ -0,0 +1,326 @@
+"""BM25 postings in array (CSR) form, and their on-disk cache.
+
+Audit follow-up 2026-09-21 (Optimization 5, 6 and 8). The postings were a
+dict of Python lists of (chunk, tf) tuples -- 7.04 M tuples over 62,016
+terms, about +522 MB of RSS -- rebuilt from 137,894 tokenize calls on every
+start (8.5-18 s of LegislationIndex()'s 12-31 s), and scored one tuple at a
+time in Python. Here each term's postings are two slices of two arrays
+(chunk indices ascending, term frequencies), so the scorer in
+canlex/index.py adds a whole term at once with numpy, and the arrays -- with
+the corpus statistics and the typo corrector's word counts -- are cached
+under data/cache/ (gitignored).
+
+The cache key is the processed files' (name, size, mtime_ns) AND a
+fingerprint of the code that produced the arrays (code_fingerprint): keyed
+by data alone it would keep serving stale postings after a tokenizer or
+stemmer change. A cache that cannot be read or written is rebuilt in
+process -- retrieval never depends on it.
+"""
+import hashlib
+import inspect
+import json
+import os
+import sys
+import time
+from array import array
+from collections import Counter
+from pathlib import Path
+from typing import NamedTuple
+
+import numpy as np
+
+from .config import DATA_DIR
+
+# Bump when the cached layout or the meaning of a cached array changes in a
+# way the code fingerprint cannot see (a dtype, a field's semantics).
+CACHE_VERSION = 1
+CACHE_DIR = Path(os.environ.get("CANLEX_CACHE_DIR") or DATA_DIR / "cache")
+_CACHE_PREFIX = "bm25-"
+# A writer's bm25-<key>.<pid>.tmp older than this is an orphan of a killed
+# process and save() removes it; a live writer's file is seconds old.
+_TMP_STALE_SECONDS = 600
+_SCOPES = ("en", "all", "fr")      # the corrector vocabularies (Bm25Data.words)
+
+
+class Postings:
+    """term -> (chunk indices ascending, term frequencies) in CSR form: the
+    postings of the i-th term are docs[offsets[i]:offsets[i + 1]] and
+    tfs[...] alike. Terms keep their first-seen corpus order, the order the
+    dict of lists had, so every vocabulary built by iterating them is
+    unchanged."""
+
+    __slots__ = ("terms", "offsets", "docs", "tfs", "_ids")
+
+    def __init__(self, terms, offsets, docs, tfs):
+        self.terms = terms
+        self.offsets, self.docs, self.tfs = offsets, docs, tfs
+        self._ids = {t: i for i, t in enumerate(terms)}
+
+    def span(self, term):
+        """(chunk indices, term frequencies) array views, or None."""
+        i = self._ids.get(term)
+        if i is None:
+            return None
+        a, b = int(self.offsets[i]), int(self.offsets[i + 1])
+        return self.docs[a:b], self.tfs[a:b]
+
+    def get(self, term, default=None):
+        """The chunk indices holding `term` -- len() is its document
+        frequency -- or `default`: the dict.get the dict of lists offered."""
+        hit = self.span(term)
+        return default if hit is None else hit[0]
+
+    def __contains__(self, term):
+        return term in self._ids
+
+    def __len__(self):
+        return len(self.terms)
+
+    def __iter__(self):
+        return iter(self.terms)
+
+
+class Bm25Data(NamedTuple):
+    """What build() derives from the chunks (and the cache stores): the
+    postings, each chunk's length in tokens, per-term document frequencies
+    over the whole corpus and over the English chunks, the English scope's
+    size and total length, and the typo corrector's word counts by scope
+    ('en': chunks whose lang is not 'fr'; 'all': every chunk; 'fr': the
+    French chunks' words with their accents)."""
+    postings: Postings
+    doc_len: np.ndarray
+    df: np.ndarray
+    df_en: np.ndarray
+    en_docs: int
+    en_total_len: int
+    words: dict
+
+
+def build(chunks, blob, tokenize, words, fr_words):
+    """Bm25Data for `chunks`. `blob(chunk)` is the text BM25 indexes,
+    `tokenize` turns it into terms, `words(text)` lists the corrector's
+    vocabulary words of a chunk's text and `fr_words(text)` a French
+    chunk's accented ones. A chunk is English unless its lang is 'fr' (the
+    scope of the English statistics and of the 'en' words)."""
+    ids = {}                        # term -> id, in first-seen order
+    term_id = ids.setdefault
+    col_term, col_doc, col_tf = array("i"), array("i"), array("i")
+    doc_len = array("q")
+    words_all, words_en, words_fr = Counter(), Counter(), Counter()
+    english = np.array([c.get("lang", "en") != "fr" for c in chunks], dtype=bool)
+    for idx, c in enumerate(chunks):
+        tokens = tokenize(blob(c))
+        counts = Counter(tokens)
+        doc_len.append(len(tokens))
+        col_term.extend([term_id(t, len(ids)) for t in counts])
+        col_tf.extend(counts.values())
+        col_doc.extend([idx] * len(counts))
+        found = words(c["text"])
+        words_all.update(found)
+        if english[idx]:
+            words_en.update(found)
+        else:
+            words_fr.update(fr_words(c["text"]))
+    postings, df, df_en = _csr(list(ids), col_term, col_doc, col_tf, english)
+    lengths = np.frombuffer(doc_len, dtype=np.int64).copy()
+    return Bm25Data(postings, lengths, df, df_en, int(english.sum()),
+                    int(lengths[english].sum()),
+                    {"en": words_en, "all": words_all, "fr": words_fr})
+
+
+def _csr(terms, col_term, col_doc, col_tf, english):
+    """(Postings, df, df_en) from the (term, chunk, tf) columns, which run
+    in chunk order: a stable sort by term keeps each term's chunks
+    ascending, as the dict of lists appended them."""
+    term = np.frombuffer(col_term, dtype=np.intc)
+    doc = np.frombuffer(col_doc, dtype=np.intc)
+    order = np.argsort(term, kind="stable")
+    df = np.bincount(term, minlength=len(terms)).astype(np.int64)
+    df_en = np.bincount(term[english[doc]], minlength=len(terms)).astype(np.int64)
+    offsets = np.zeros(len(terms) + 1, dtype=np.int64)
+    np.cumsum(df, out=offsets[1:])
+    tfs = np.frombuffer(col_tf, dtype=np.intc)[order]
+    if tfs.size and int(tfs.max()) < 2 ** 16:
+        tfs = tfs.astype(np.uint16)
+    docs = doc[order].astype(np.int32)
+    return Postings(terms, offsets, docs, tfs), df, df_en
+
+
+def first_seen(parts):
+    """The distinct values of the int arrays `parts`, in order of first
+    appearance -- the insertion order a dict filled term by term had."""
+    if not parts:
+        return np.zeros(0, dtype=np.int64)
+    flat = np.concatenate(parts)
+    uniq, first = np.unique(flat, return_index=True)
+    return uniq[np.argsort(first, kind="stable")]
+
+
+# ---- the cache ---------------------------------------------------------------
+
+def _source(obj):
+    """What of `obj` the fingerprint hashes: a function's source (its code
+    object when no source is available), anything else's repr."""
+    if callable(obj):
+        try:
+            return inspect.getsource(obj)
+        except (OSError, TypeError):
+            code = getattr(obj, "__code__", None)
+            return repr((code.co_code, code.co_consts)) if code else repr(obj)
+    return repr(obj)
+
+
+def code_fingerprint(*parts):
+    """sha256 over CACHE_VERSION, build()'s own source and every part (the
+    tokenizer, the stemmer and its table, the blob and word functions, the
+    stemmer's version -- whatever shapes the cached arrays)."""
+    h = hashlib.sha256()
+    for part in (CACHE_VERSION, build, _csr, *parts):
+        h.update(_source(part).encode("utf-8"))
+        h.update(b"\x00")
+    return h.hexdigest()
+
+
+def file_stamps(paths):
+    """[(name, size, mtime_ns)] of the processed files, in the given order."""
+    out = []
+    for p in paths:
+        st = Path(p).stat()
+        out.append((Path(p).name, st.st_size, st.st_mtime_ns))
+    return out
+
+
+def cache_key(stamps, code):
+    """The cache key: the data stamps and the code fingerprint together."""
+    return json.dumps({"version": CACHE_VERSION, "code": code, "files": stamps},
+                      separators=(",", ":"))
+
+
+def cache_path(key, cache_dir=None):
+    digest = hashlib.sha256(key.encode("utf-8")).hexdigest()[:16]
+    return Path(cache_dir or CACHE_DIR) / f"{_CACHE_PREFIX}{digest}.npz"
+
+
+def _joined(strings):
+    return np.frombuffer("\n".join(strings).encode("utf-8"), dtype=np.uint8)
+
+
+def _split(arr):
+    text = arr.tobytes().decode("utf-8")
+    return text.split("\n") if text else []
+
+
+def _drop_stale_tmp(directory):
+    """Remove the bm25-*.tmp files in `directory` last written more than
+    _TMP_STALE_SECONDS ago: what a writer killed between open() and
+    os.replace() leaves. A file that cannot be read or removed is left.
+    Verification 2026-09-22 (A-CACHE-TMP): save() swept only *.npz, so each
+    killed writer's partial cache stayed on disk for good."""
+    now = time.time()
+    for tmp in Path(directory).glob(f"{_CACHE_PREFIX}*.tmp"):
+        try:
+            if now - tmp.stat().st_mtime > _TMP_STALE_SECONDS:
+                tmp.unlink()
+        except OSError:
+            pass            # gone already, or still held open by its writer
+
+
+def save(key, data, cache_dir=None):
+    """Write `data` under `key` atomically and drop the directory's other
+    BM25 caches and orphaned partial writes (_drop_stale_tmp); False, with
+    a note on stderr, when it cannot be written."""
+    path = cache_path(key, cache_dir)
+    tmp = path.with_name(f"{path.stem}.{os.getpid()}.tmp")
+    p = data.postings
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        _drop_stale_tmp(path.parent)
+        with open(tmp, "wb") as fh:
+            np.savez(fh, key=np.array(key), terms=_joined(p.terms),
+                     offsets=p.offsets, docs=p.docs, tfs=p.tfs,
+                     doc_len=data.doc_len, df=data.df, df_en=data.df_en,
+                     scope=np.array([data.en_docs, data.en_total_len], dtype=np.int64),
+                     **{f"words_{s}": _joined(data.words[s]) for s in _SCOPES},
+                     **{f"counts_{s}": np.fromiter(data.words[s].values(), np.int64,
+                                                   len(data.words[s]))
+                        for s in _SCOPES})
+        os.replace(tmp, path)
+    except OSError as exc:
+        print(f"CanLex index: BM25 cache not written ({exc}); "
+              f"built in process.", file=sys.stderr)
+        tmp.unlink(missing_ok=True)
+        return False
+    for old in path.parent.glob(f"{_CACHE_PREFIX}*.npz"):
+        if old != path:
+            try:
+                old.unlink()
+            except OSError:
+                pass            # another process may still be reading it
+    return True
+
+
+def _consistent(n_chunks, n_terms, offsets, docs, tfs, doc_len, df, df_en):
+    """Whether cached arrays can index a corpus of `n_chunks` chunks and
+    `n_terms` terms: integer arrays; offsets from 0, never decreasing, up to
+    len(docs); one tf per posting; every posting's chunk index inside the
+    corpus; one length per chunk; one df and one df_en per term.
+    Verification 2026-09-22 (SEC-3): a cache whose postings named chunks
+    999999 and -5 was accepted -- an IndexError on every search, or a
+    silently wrong ranking (-5 is the fifth-last chunk)."""
+    arrays = (offsets, docs, tfs, doc_len, df, df_en)
+    if any(a.ndim != 1 or a.dtype.kind not in "iu" for a in arrays):
+        return False
+    if (len(offsets) != n_terms + 1 or int(offsets[0]) != 0
+            or int(offsets[-1]) != len(docs) or np.any(np.diff(offsets) < 0)):
+        return False
+    if len(docs) != len(tfs) or len(doc_len) != n_chunks:
+        return False
+    if docs.size and (int(docs.min()) < 0 or int(docs.max()) >= n_chunks):
+        return False
+    return len(df) == n_terms and len(df_en) == n_terms
+
+
+def load(key, n_chunks, cache_dir=None):
+    """The cached Bm25Data for `key`, or None when there is none, it is
+    empty, it was written for another key or corpus size, its arrays do
+    not fit together (_consistent), or it cannot be read.
+
+    A damaged file raises many types inside np.load -- EOFError on zero
+    bytes, BadZipFile on a truncated one, NotImplementedError on a flipped
+    compression field, KeyError, ValueError -- and every one of them means
+    the same: rebuild. Verification 2026-09-22 (A-CACHE-EOF): a zero-byte
+    cache raised EOFError, which the except clause did not name, so
+    LegislationIndex() and every search tool failed on each start and
+    nothing ever rewrote the file."""
+    path = cache_path(key, cache_dir)
+    if not path.exists():
+        return None
+    try:
+        if path.stat().st_size == 0:
+            print("CanLex index: BM25 cache empty (an interrupted write); "
+                  "rebuilding.", file=sys.stderr)
+            return None
+        with np.load(path, allow_pickle=False) as z:
+            if z["key"].item() != key or len(z["doc_len"]) != n_chunks:
+                return None
+            terms = _split(z["terms"])
+            offsets, docs, tfs = z["offsets"], z["docs"], z["tfs"]
+            doc_len, df, df_en = z["doc_len"], z["df"], z["df_en"]
+            words = {s: (_split(z[f"words_{s}"]), z[f"counts_{s}"].tolist())
+                     for s in _SCOPES}
+            scope = z["scope"].tolist()
+        if (not _consistent(n_chunks, len(terms), offsets, docs, tfs, doc_len,
+                            df, df_en)
+                or len(scope) != 2
+                or any(len(w) != len(c) for w, c in words.values())):
+            print("CanLex index: BM25 cache inconsistent; rebuilding.",
+                  file=sys.stderr)
+            return None
+        en_docs, en_total = scope
+        return Bm25Data(Postings(terms, offsets, docs, tfs), doc_len, df, df_en,
+                        en_docs, en_total,
+                        {s: Counter(dict(zip(w, c))) for s, (w, c) in words.items()})
+    except Exception as exc:        # noqa: BLE001 -- any damage means rebuild
+        print(f"CanLex index: BM25 cache unreadable ({type(exc).__name__}); "
+              f"rebuilding.", file=sys.stderr)
+        return None

canlex/calllog.py

@@ -0,0 +1,833 @@
+"""The MCP call log (2026-09-16): one record per tool call, opt-in, best-effort.
+
+The maintainer wants to see what people ask CanLex through the MCP tools.
+install() wraps the FastMCP tool manager's call_tool ONCE at startup, so every
+tool call -- whichever tool, whether it returns or raises -- yields one JSON
+record. No tool function is touched (the review gate fingerprints those, see
+canlex/reviewed_code.py); nothing here carries legal wording or routing.
+
+A record holds:
+  ts            UTC, ISO 8601
+  tool          the tool name the client called
+  arguments     the arguments exactly as the client sent them; a string over
+                MAX_STRING characters is cut and marked, and its path is listed
+                in 'truncated'
+  duration_ms, ok, output_chars
+  error         the exception's TYPE NAME only (never its message or traceback)
+  error_id      for a tool's own unexpected exception: the reference id the
+                client was given and the stderr traceback carries (the error
+                guard, audit follow-up 2026-09-21)
+  n_results, confidence, top_citations
+                canlex_search_legislation only (server._log_query -> note())
+  result        branch / rehab_status / status / verdict / needs, read from a
+                tool's trailing CANLEX_RESULT line (the screening tools)
+  dropped_before
+                how many calls went unrecorded (queue full) before this one
+A record never holds an IP address, a header, a user agent, a session id or
+any auth material: the wrapper sees only the tool name and its arguments, and
+the record is built from nothing else.
+
+Sinks, each turned on by the environment (nothing set = no wrapper, no thread,
+no logging, no notice -- the local stdio server is unchanged):
+  CANLEX_LOG_DIR      local JSONL, appended to CANLEX_LOG_DIR/calls.jsonl
+  CANLEX_LOG_DATASET and CANLEX_LOG_TOKEN (BOTH)
+                      durable: huggingface_hub.CommitScheduler pushes
+                      logs/YYYY-MM-DD/<uuid>.jsonl to that dataset repo every
+                      CANLEX_LOG_EVERY_MIN minutes (default 10), and once more
+                      at shutdown. The dataset must be private: one that is
+                      not is never pushed to. Half-configured, a token the Hub
+                      refuses, or a public dataset: one stderr warning, and
+                      serving continues.
+
+The notice: while logging is on, the server tells MCP clients so in its MCP
+`instructions` (NOTICE, a plain statement that gives no advice about what to
+send) and answers GET / on the HTTP transport with the same statement.
+
+The error guard (audit follow-up 2026-09-21, missing 8): FastMCP answers a
+tool's own exception with str(exc) ('int' object has no attribute 'lower'), and
+the record keeps only the type, so the maintainer saw neither message nor
+stack. The wrapper now prints the traceback to stderr under a fresh uuid, notes
+the uuid in the record (error_id) and answers the client with one line naming
+the exception's type and the reference. An unknown tool, arguments that failed
+validation (the caller's own words, which it needs to see) and a cancellation
+pass through unchanged.
+
+Best-effort throughout: the wrapper only puts a small tuple on a bounded queue
+(put_nowait -- on overflow the call is dropped and counted); a background
+thread builds the records and writes them. A sink that raises or hangs cannot
+slow, change or break a tool response; a batch a sink refuses is retried one
+record at a time, and a sink's failures are reported on stderr once per error
+type and then at most once every WARN_EVERY seconds. A lone UTF-16 surrogate
+in an argument (valid JSON, not UTF-8) is recorded as its backslash escape.
+The token is never printed or logged.
+"""
+import atexit
+import contextvars
+import datetime
+import json
+import os
+import queue
+import sys
+import tempfile
+import threading
+import time
+import traceback
+import uuid
+from pathlib import Path
+
+LOG_FILE = "calls.jsonl"
+MAX_STRING = 4000            # a longer argument string is cut and marked
+MAX_ITEMS = 1000             # a longer argument list or object is cut and marked
+ARGS_BUDGET = 64000          # characters of argument text one record keeps in all
+# (the code audit of 2026-09-21: MAX_STRING and MAX_ITEMS bound each node, not the
+# record -- a wide extra key of 1,000 4,000-character strings passed validation, since
+# FastMCP ignores unknown top-level keys, and wrote a 4 MB line; past the budget the
+# rest of the arguments is dropped and the path noted in `truncated`)
+QUEUE_SIZE = 1000            # calls waiting for the writer; beyond it, dropped
+PENDING_CAP = 10000          # dataset lines held while the scheduler starts
+DEFAULT_EVERY_MIN = 10.0
+PATH_IN_REPO = "logs"
+RETRY_DELAYS = (30, 120, 600)    # seconds between attempts to open the dataset
+RESULT_MARK = "CANLEX_RESULT: "
+RESULT_KEYS = ("branch", "rehab_status", "status", "verdict", "needs",
+               # step 1 ids (disposition roadmap T4.4, the review of batch 1): which
+               # entry answered, the footing, how the act branch stands, the wording flags
+               "disposition_entry", "entry", "footing", "act_branch", "question_flag",
+               "disposition_flags", "allegation_paragraphs")
+SEARCH_KEYS = ("n_results", "confidence", "top_citations")
+ERROR_KEYS = ("error_id",)
+NEEDS_CAP = 10
+NEED_CHARS = 500
+WARN_EVERY = 3600.0          # seconds between repeats of one sink's same failure warning
+NOTICE = ("Tool calls to this server, including their arguments, are logged by "
+          "the operator.")
+
+_STOP = object()
+_extra = contextvars.ContextVar("canlex_call_extra", default=None)
+
+
+def _warn(message):
+    try:
+        print(f"CanLex call log: {message}", file=sys.stderr, flush=True)
+    except Exception:
+        pass
+
+
+def note(**fields):
+    """Attach search fields (n_results, confidence, top_citations) or the
+    error guard's error_id to the call being recorded. A no-op when no call
+    is being recorded."""
+    extra = _extra.get()
+    if extra is not None:
+        extra.update((k, v) for k, v in fields.items() if k in SEARCH_KEYS + ERROR_KEYS)
+
+
+def recording():
+    """True inside a tool call the call log is recording."""
+    return _extra.get() is not None
+
+
+# ---- building a record (writer thread) --------------------------------------
+
+def _iso(ts):
+    return (datetime.datetime.fromtimestamp(ts, datetime.timezone.utc)
+            .isoformat(timespec="milliseconds").replace("+00:00", "Z"))
+
+
+def _cut(text, path, cut):
+    """A string for the record: cut past MAX_STRING characters (and marked),
+    and always encodable as UTF-8 -- a lone UTF-16 surrogate, which a client
+    can send as valid JSON (a string cut inside an emoji), becomes its
+    backslash escape, so the line it is on can be written."""
+    if len(text) > MAX_STRING:
+        cut.append(path)
+        text = text[:MAX_STRING] + f" [truncated: {len(text)} chars]"
+    if not text.isascii():
+        try:
+            text.encode("utf-8")
+        except UnicodeEncodeError:
+            text = text.encode("utf-8", "backslashreplace").decode("utf-8")
+    return text
+
+
+def _clean(value, path, cut, depth=0, budget=None):
+    """A JSON-serialisable copy of an argument value: strings over MAX_STRING
+    characters and containers over MAX_ITEMS items are cut, their paths listed
+    in `cut`; anything that is not JSON is recorded as its repr. `budget` is
+    the characters the whole record may still keep (ARGS_BUDGET at the top):
+    every string, key and scalar is charged to it, and once it is spent the
+    rest is dropped -- a container stops where it stands and its path is
+    noted in `cut` once."""
+    if budget is None:
+        budget = [ARGS_BUDGET]
+
+    def spent(where):
+        if budget[0] <= 0:
+            if where not in cut:
+                cut.append(where)
+            return True
+        return False
+
+    def charge(text):
+        # the text this record keeps, cut to what the budget still allows
+        if len(text) > budget[0]:
+            kept = text[:max(budget[0], 0)] + f" [truncated: {len(text)} chars]"
+            budget[0] = 0
+            if path not in cut:
+                cut.append(path)
+            return kept
+        budget[0] -= len(text)
+        return text
+
+    if isinstance(value, str):
+        if spent(path):
+            return "[truncated: record budget]"
+        return charge(_cut(value, path, cut))
+    if value is None or isinstance(value, (bool, int)):
+        budget[0] -= len(str(value))
+        return value
+    if isinstance(value, float):
+        budget[0] -= len(repr(value))
+        return value if value == value and abs(value) != float("inf") else repr(value)
+    if depth >= 32:
+        cut.append(path)
+        return "[nested too deep]"
+    if isinstance(value, dict):
+        items = list(value.items())
+        if len(items) > MAX_ITEMS:
+            cut.append(path)
+            items = items[:MAX_ITEMS]
+        out = {}
+        for k, v in items:
+            if spent(path):
+                break
+            key = charge(_cut(str(k), f"{path}.<key>", cut))
+            out[key] = _clean(v, f"{path}.{key[:40]}", cut, depth + 1, budget)
+        return out
+    if isinstance(value, (list, tuple)):
+        items = list(value)
+        if len(items) > MAX_ITEMS:
+            cut.append(path)
+            items = items[:MAX_ITEMS]
+        out = []
+        for i, v in enumerate(items):
+            if spent(path):
+                break
+            out.append(_clean(v, f"{path}[{i}]", cut, depth + 1, budget))
+        return out
+    if spent(path):
+        return "[truncated: record budget]"
+    return charge(_cut(repr(value), path, cut))
+
+
+def _output_text(result):
+    """The text a tool call returned: FastMCP hands back a content list, a
+    (content list, structured) tuple, or a CallToolResult."""
+    if isinstance(result, tuple) and result:
+        result = result[0]
+    content = getattr(result, "content", result)
+    if isinstance(content, str):
+        return content
+    if isinstance(content, (list, tuple)):
+        return "".join(t for t in (getattr(b, "text", None) for b in content)
+                       if isinstance(t, str))
+    return ""
+
+
+def _result_line(text):
+    """The JSON after the last CANLEX_RESULT marker, unparsed ('' if none)."""
+    at = text.rfind(RESULT_MARK)
+    if at < 0:
+        return ""
+    return text[at + len(RESULT_MARK):].split("\n", 1)[0]
+
+
+def _need(item):
+    if isinstance(item, dict):
+        item = item.get("question") or item.get("id") or item.get("parameter") or item
+    text = item if isinstance(item, str) else json.dumps(item, ensure_ascii=False,
+                                                         default=repr)
+    return text[:NEED_CHARS]
+
+
+def _result_fields(line):
+    """branch / rehab_status / status / verdict / needs from a CANLEX_RESULT
+    line, or None when there is none or it does not parse."""
+    if not line:
+        return None
+    try:
+        data = json.loads(line)
+    except ValueError:
+        return None
+    if not isinstance(data, dict):
+        return None
+    out = {}
+    for key in RESULT_KEYS:
+        if key not in data:
+            continue
+        value = data[key]
+        if key == "needs":
+            value = ([_need(n) for n in value[:NEEDS_CAP]] if isinstance(value, list)
+                     else None)
+        out[key] = _clean(value, key, [])
+    return out or None
+
+
+def _error_name(exc):
+    """The raised exception's type name; FastMCP wraps a tool's own exception
+    in ToolError, so the wrapped one is named when there is one."""
+    cause = exc.__cause__
+    if type(exc).__name__ == "ToolError" and cause is not None:
+        exc = cause
+    return type(exc).__name__
+
+
+def build_record(started, tool, arguments, duration_ms, error, output_chars,
+                 result_line, extra):
+    """One call's record (see the module docstring for the fields)."""
+    cut = []
+    record = {
+        "ts": _iso(started),
+        "tool": _cut(str(tool), "tool", cut),
+        "arguments": _clean(arguments if arguments is not None else {}, "arguments", cut),
+        "duration_ms": round(duration_ms, 1),
+        "ok": error is None,
+    }
+    if error is not None:
+        record["error"] = _cut(str(error), "error", cut)
+    record["output_chars"] = output_chars
+    for key in SEARCH_KEYS + ERROR_KEYS:
+        if extra and key in extra:
+            record[key] = _clean(extra[key], key, cut)
+    fields = _result_fields(result_line)
+    if fields:
+        record["result"] = fields
+    if cut:
+        record["truncated"] = cut
+    return record
+
+
+# ---- the queue and the writer thread ---------------------------------------
+
+class CallLog:
+    """A bounded queue of tool calls and the background thread that turns
+    them into records and hands them to the sinks."""
+
+    def __init__(self, sinks, maxsize=QUEUE_SIZE):
+        self.sinks = list(sinks)
+        self.dropped = 0           # calls not recorded because the queue was full
+        self.written = 0           # records at least one sink accepted
+        self.unwritten = 0         # records built that no sink accepted
+        self._unreported = 0       # drops not yet carried on a record
+        self._lock = threading.Lock()
+        self._warned = {}          # (what, error type) -> [last reported, failures since]
+        self._closed = False
+        self._queue = queue.Queue(maxsize=maxsize)
+        self._thread = threading.Thread(target=self._run, name="canlex-call-log",
+                                        daemon=True)
+        self._thread.start()
+
+    def submit(self, *call):
+        """Queue one call without ever blocking; False when it was dropped."""
+        if self._closed:
+            return False
+        try:
+            self._queue.put_nowait(call)
+            return True
+        except queue.Full:
+            with self._lock:
+                self.dropped += 1
+                self._unreported += 1
+        except Exception:
+            pass
+        return False
+
+    def _run(self):
+        while True:
+            batch, stop = [], False
+            item = self._queue.get()
+            while True:
+                if item is _STOP:
+                    stop = True
+                    break
+                batch.append(item)
+                if len(batch) >= 200:
+                    break
+                try:
+                    item = self._queue.get_nowait()
+                except queue.Empty:
+                    break
+            if batch:
+                self._write(batch)
+            if stop:
+                return
+
+    def _write(self, batch):
+        records, lines = [], []
+        for call in batch:
+            try:
+                record = build_record(*call)
+                line = json.dumps(record, ensure_ascii=False)
+            except Exception as exc:
+                self._report("record", "a record could not be built", exc)
+                continue
+            with self._lock:
+                if self._unreported:
+                    record["dropped_before"] = self._unreported
+                    self._unreported = 0
+                    line = json.dumps(record, ensure_ascii=False)
+            lines.append(line)
+            records.append(record)
+        if not records:
+            return
+        stored = [False] * len(records)
+        for sink in self.sinks:
+            try:
+                sink.write(records, lines)
+                stored = [True] * len(records)
+                continue
+            except Exception as exc:
+                self._report(id(sink), f"the {sink.name} failed to write", exc)
+            # one bad record (or a transient fault) must not cost the whole
+            # batch, other people's calls included: retry one record at a time
+            for i, (record, line) in enumerate(zip(records, lines)):
+                try:
+                    sink.write([record], [line])
+                    stored[i] = True
+                except Exception as exc:
+                    self._report(id(sink), f"the {sink.name} failed to write", exc)
+        kept = sum(stored)
+        self.written += kept
+        self.unwritten += len(records) - kept
+
+    def _report(self, what, message, exc):
+        """A failure on stderr, by exception type only: the first of each
+        type at once, then at most once every WARN_EVERY seconds, with how
+        many went unreported in between."""
+        key, now = (what, type(exc).__name__), time.monotonic()
+        seen = self._warned.get(key)
+        if seen is not None and now - seen[0] < WARN_EVERY:
+            seen[1] += 1
+            return
+        since = seen[1] if seen is not None else 0
+        self._warned[key] = [now, 0]
+        more = f"; {since} more since the last report" if since else ""
+        _warn(f"{message} ({type(exc).__name__}){more}; a repeat is reported at most "
+              f"every {WARN_EVERY / 60:g} min")
+
+    def close(self, timeout=5.0):
+        """Write what is queued, then close the sinks (the dataset sink's
+        final push). Safe to call more than once; run at interpreter exit."""
+        if self._closed:
+            return
+        self._closed = True
+        try:
+            self._queue.put(_STOP, timeout=timeout)
+            self._thread.join(timeout)
+        except Exception:
+            pass
+        for sink in self.sinks:
+            try:
+                sink.close()
+            except Exception:
+                pass
+
+
+# ---- sinks ------------------------------------------------------------------
+
+class LocalSink:
+    """Appends each record to CANLEX_LOG_DIR/calls.jsonl."""
+    name = "local call log"
+
+    def __init__(self, directory):
+        self.path = Path(directory) / LOG_FILE
+
+    def write(self, records, lines):
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("a", encoding="utf-8") as fh:
+            fh.write("".join(line + "\n" for line in lines))
+
+    def close(self):
+        pass
+
+
+def _status(exc):
+    return getattr(getattr(exc, "response", None), "status_code", None)
+
+
+def _describe(exc):
+    """An exception's type name and HTTP status -- never its message, which
+    could echo a request."""
+    status = _status(exc)
+    return type(exc).__name__ + (f", HTTP {status}" if status else "")
+
+
+class DatasetSink:
+    """Pushes the records to a private Hugging Face dataset repo with
+    huggingface_hub.CommitScheduler: this process writes
+    <folder>/YYYY-MM-DD/<uuid>.jsonl, the scheduler uploads the folder to
+    logs/ every `every` minutes. The scheduler is opened on a background
+    thread (it reaches the Hub), so startup never waits on the network; lines
+    written before it is up are held in memory (up to PENDING_CAP)."""
+    name = "dataset call log"
+
+    def __init__(self, repo_id, token, every=DEFAULT_EVERY_MIN, folder=None):
+        self.repo_id = repo_id
+        self.every = every
+        self._token = token
+        self.file_id = uuid.uuid4().hex
+        self.folder = (Path(folder) if folder else
+                       Path(tempfile.gettempdir()) / "canlex-call-log" / self.file_id)
+        self.scheduler = None
+        self.failed = False
+        self.ready = threading.Event()     # set once the scheduler is up or has failed
+        self._pending = []
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._closed = False
+        self._thread = None
+
+    def __repr__(self):
+        return f"<DatasetSink {self.repo_id}>"
+
+    def start(self):
+        """Open the scheduler on a background thread. Under HF_HUB_OFFLINE the
+        Hub client is replaced first, here, before the server takes requests
+        (see allow_dataset_requests), so a model load can never be using the
+        client that set_client_factory closes."""
+        if self._thread is not None:
+            return
+        try:
+            from huggingface_hub import constants
+            if constants.is_offline_mode():
+                allow_dataset_requests(self.repo_id)
+        except Exception:
+            pass        # the scheduler's first request then fails and says so
+        self._thread = threading.Thread(target=self._open, name="canlex-call-log-hub",
+                                        daemon=True)
+        self._thread.start()
+
+    def _open(self):
+        last, public = None, False
+        for delay in (0,) + tuple(RETRY_DELAYS):
+            if delay and self._stop.wait(delay):
+                break
+            scheduler = None
+            try:
+                from huggingface_hub import CommitScheduler
+                scheduler = CommitScheduler(
+                    repo_id=self.repo_id, repo_type="dataset", folder_path=self.folder,
+                    path_in_repo=PATH_IN_REPO, every=self.every, private=True,
+                    token=self._token, allow_patterns=["*.jsonl"])
+                # private=True only applies when the scheduler CREATES the repo;
+                # an existing public dataset would publish every record. Checked
+                # before any record reaches the scheduler's folder.
+                info = scheduler.api.repo_info(self.repo_id, repo_type="dataset")
+                public = getattr(info, "private", None) is not True
+            except Exception as exc:
+                self._halt(scheduler)
+                last = exc
+                status = _status(exc)
+                retry = (not isinstance(exc, (ImportError, ValueError, TypeError))
+                         and (status is None or status == 429 or status >= 500))
+                if not retry:
+                    break
+                continue
+            if public:
+                self._halt(scheduler)
+                break
+            with self._lock:
+                self.scheduler = scheduler
+                pending, self._pending = self._pending, []
+                self._append(pending)
+            _warn(f"the dataset call log is open: {self.repo_id!r} (private); records "
+                  f"are pushed every {self.every:g} min")
+            self.ready.set()
+            return
+        self.failed = True
+        with self._lock:
+            self._pending = []
+        if public:
+            _warn(f"the durable call log is off: the dataset {self.repo_id!r} is not "
+                  "private, so nothing is pushed to it -- make it private in its "
+                  "settings and restart; tool calls are still served")
+        else:
+            hint = (" -- check CANLEX_LOG_TOKEN (a fine-grained token with read and write "
+                    "access to that dataset) and CANLEX_LOG_DATASET") if last is not None else ""
+            _warn(f"the durable call log is off: could not open the dataset "
+                  f"{self.repo_id!r} ({_describe(last) if last else 'stopped'}){hint}; "
+                  "tool calls are still served")
+        self.ready.set()
+
+    @staticmethod
+    def _halt(scheduler):
+        """Stop a scheduler that must not push (its folder is still empty)."""
+        if scheduler is None:
+            return
+        try:
+            scheduler.stop()
+        except Exception:
+            pass
+
+    def write(self, records, lines):
+        if self.failed or self._closed:
+            return
+        items = [(r["ts"][:10], line) for r, line in zip(records, lines)]
+        with self._lock:
+            if self.scheduler is None:
+                room = max(PENDING_CAP - len(self._pending), 0)
+                self._pending.extend(items[:room])
+                return
+            self._append(items)
+
+    def _append(self, items):
+        if not items:
+            return
+        by_day = {}
+        for day, line in items:
+            by_day.setdefault(day, []).append(line)
+        with self.scheduler.lock:
+            for day, day_lines in by_day.items():
+                path = self.folder / day / f"{self.file_id}.jsonl"
+                path.parent.mkdir(parents=True, exist_ok=True)
+                with path.open("a", encoding="utf-8") as fh:
+                    fh.write("".join(line + "\n" for line in day_lines))
+
+    def close(self):
+        """The final push at shutdown (best-effort)."""
+        if self._closed:
+            return
+        self._closed = True
+        self._stop.set()
+        scheduler = self.scheduler
+        if scheduler is None:
+            return
+        try:
+            scheduler.push_to_hub()
+        except Exception as exc:
+            _warn(f"the final push to {self.repo_id!r} failed ({_describe(exc)})")
+        try:
+            scheduler.stop()
+        except Exception:
+            pass
+
+
+def allow_dataset_requests(repo_id):
+    """Let the call log reach its dataset under HF_HUB_OFFLINE=1.
+
+    The Space's Dockerfile sets HF_HUB_OFFLINE=1 so the embedder and reranker
+    are only ever served from the model cache baked into the image. That flag
+    blocks EVERY huggingface_hub request, the scheduler's pushes included, in
+    the library's request hook. This installs a client (huggingface_hub's
+    public set_client_factory; otherwise the library default) whose hook
+    still refuses every Hub request EXCEPT repo creation and requests for
+    this dataset repo. Requests to other hosts pass: they are storage URLs a
+    Hub response handed out, and a model download cannot get one, because its
+    first request (to the Hub's /resolve/ path) is still refused."""
+    import httpx
+    from huggingface_hub import constants, set_client_factory
+    from huggingface_hub.errors import OfflineModeIsEnabled
+
+    hub_host = httpx.URL(constants.ENDPOINT).host
+    repo = repo_id.strip("/").lower()
+    exact = {f"/api/datasets/{repo}", "/api/repos/create"}
+    prefixes = (f"/api/datasets/{repo}/", f"/datasets/{repo}/", f"/datasets/{repo}.git/")
+
+    def allowed(url):
+        if url.host != hub_host:
+            return True
+        path = url.path.lower()
+        return path in exact or path.startswith(prefixes)
+
+    def hook(request):
+        if constants.is_offline_mode() and not allowed(request.url):
+            raise OfflineModeIsEnabled(
+                f"Cannot reach {request.url}: offline mode is enabled. To disable it, "
+                "please unset the `HF_HUB_OFFLINE` environment variable.")
+        if "X-Amzn-Trace-Id" not in request.headers:
+            request.headers["X-Amzn-Trace-Id"] = (request.headers.get("x-request-id")
+                                                  or str(uuid.uuid4()))
+
+    def factory():
+        return httpx.Client(event_hooks={"request": [hook]}, follow_redirects=True,
+                            timeout=None)
+
+    set_client_factory(factory)
+    return factory
+
+
+# ---- configuration and the wrapper -----------------------------------------
+
+def _every(env):
+    raw = (env.get("CANLEX_LOG_EVERY_MIN") or "").strip()
+    if not raw:
+        return DEFAULT_EVERY_MIN
+    try:
+        value = float(raw)
+        if value > 0 and value == value and value != float("inf"):
+            return value
+    except ValueError:
+        pass
+    _warn(f"CANLEX_LOG_EVERY_MIN={raw!r} is not a positive number of minutes; "
+          f"using {DEFAULT_EVERY_MIN:g}")
+    return DEFAULT_EVERY_MIN
+
+
+def configure(environ=None, dataset_folder=None):
+    """The sinks the environment turns on ([] = logging off). A dataset sink
+    needs BOTH CANLEX_LOG_DATASET and CANLEX_LOG_TOKEN; one without the other
+    gets one stderr warning (which never includes the token)."""
+    env = os.environ if environ is None else environ
+    sinks = []
+    log_dir = (env.get("CANLEX_LOG_DIR") or "").strip()
+    if log_dir:
+        sinks.append(LocalSink(log_dir))
+    dataset = (env.get("CANLEX_LOG_DATASET") or "").strip()
+    token = (env.get("CANLEX_LOG_TOKEN") or "").strip()
+    if dataset and token:
+        sinks.append(DatasetSink(dataset, token, _every(env), folder=dataset_folder))
+    elif dataset:
+        _warn("CANLEX_LOG_DATASET is set but CANLEX_LOG_TOKEN is not: the durable call "
+              "log is off; tool calls are still served")
+    elif token:
+        _warn("CANLEX_LOG_TOKEN is set but CANLEX_LOG_DATASET is not: the durable call "
+              "log is off; tool calls are still served")
+    return sinks
+
+
+def _guard(name, exc):
+    """The ToolError the client gets instead of `exc`, or None to re-raise
+    `exc` unchanged (the error guard, audit follow-up 2026-09-21). Only a
+    tool's own exception is guarded -- FastMCP's ToolError whose cause is an
+    Exception other than a pydantic ValidationError: its traceback goes to
+    stderr under a fresh uuid, the uuid goes into the record (note), and the
+    client reads 'Error: CanLex could not complete this call (<type>);
+    reference <uuid>'. Never raises itself."""
+    try:
+        from mcp.server.fastmcp.exceptions import ToolError
+        from pydantic import ValidationError
+        cause = exc.__cause__
+        if (not isinstance(exc, ToolError) or not isinstance(cause, Exception)
+                or isinstance(cause, ValidationError)):
+            return None
+        ref = str(uuid.uuid4())
+        stack = "".join(traceback.format_exception(type(cause), cause, cause.__traceback__))
+        print(f"CanLex MCP: tool {_cut(str(name), 'tool', [])} failed; reference {ref}\n"
+              f"{stack}", file=sys.stderr, flush=True)
+        note(error_id=ref)
+        return ToolError(f"Error: CanLex could not complete this call "
+                         f"({type(cause).__name__}); reference {ref}")
+    except Exception:
+        return None
+
+
+def wrap(manager, log):
+    """Wrap a FastMCP ToolManager's call_tool (mcp 1.27.1: FastMCP.call_tool ->
+    ToolManager.call_tool(name, arguments, context=, convert_result=) ->
+    Tool.run) so every call is submitted to `log`, and a tool's own exception
+    reaches the client as a reference id (_guard). Once per manager."""
+    if getattr(manager, "_canlex_call_log", None) is not None:
+        return manager
+    original = manager.call_tool
+
+    async def call_tool(name, arguments, context=None, convert_result=False):
+        started, clock = time.time(), time.perf_counter()
+        extra = {}
+        context_token = _extra.set(extra)
+        error, result = None, None
+        try:
+            result = await original(name, arguments, context=context,
+                                    convert_result=convert_result)
+            return result
+        except Exception as exc:
+            error = exc
+            guarded = _guard(name, exc)
+            if guarded is None:
+                raise
+            raise guarded from exc
+        except BaseException as exc:
+            error = exc
+            raise
+        finally:
+            try:
+                _extra.reset(context_token)
+            except Exception:
+                pass
+            try:
+                duration = (time.perf_counter() - clock) * 1000
+                if error is None:
+                    text = _output_text(result)
+                    log.submit(started, name, arguments, duration, None, len(text),
+                               _result_line(text), extra)
+                else:
+                    log.submit(started, name, arguments, duration, _error_name(error), 0,
+                               "", extra)
+            except Exception:
+                pass
+
+    manager.call_tool = call_tool
+    manager._canlex_call_log = log
+    return manager
+
+
+def announce(server):
+    """Tell the server's users that calls are logged: NOTICE in the MCP
+    `instructions` every client receives at initialize (after any existing
+    instructions), and a plain-text GET / on the HTTP transport, where the
+    Space page's frame otherwise shows a 404. A statement of fact only -- it
+    gives no advice about which arguments to send (see RUNBOOK.md, "Call
+    log"). Best-effort: a failure is one stderr line."""
+    try:
+        low = server._mcp_server
+        current = low.instructions
+        if not current:
+            low.instructions = NOTICE
+        elif NOTICE not in current:
+            low.instructions = f"{current}\n\n{NOTICE}"
+    except Exception as exc:
+        _warn(f"the notice could not be added to the MCP instructions "
+              f"({type(exc).__name__})")
+    try:
+        if any(getattr(r, "path", None) == "/" for r in server._custom_starlette_routes):
+            return
+        from starlette.responses import PlainTextResponse
+        endpoint = getattr(getattr(server, "settings", None), "streamable_http_path", "/mcp")
+        page = (f"CanLex MCP server: connect an MCP client to {endpoint} on this "
+                f"host.\n\n{NOTICE}\n")
+
+        async def root(request):
+            return PlainTextResponse(page)
+
+        server.custom_route("/", methods=["GET"])(root)
+    except Exception as exc:
+        _warn(f"the notice page at / could not be added ({type(exc).__name__})")
+
+
+def install(server, environ=None, dataset_folder=None):
+    """Turn the call log on for a FastMCP server when the environment asks
+    for it: wrap its tool manager, publish the notice (announce), start the
+    sinks, and write what is queued (and push) at interpreter exit. Returns
+    the CallLog, or None when nothing is configured -- then nothing is
+    wrapped, no notice is added and no thread starts."""
+    existing = getattr(server._tool_manager, "_canlex_call_log", None)
+    if existing is not None:
+        return existing
+    sinks = configure(environ, dataset_folder=dataset_folder)
+    if not sinks:
+        return None
+    log = CallLog(sinks)
+    wrap(server._tool_manager, log)
+    announce(server)
+    atexit.register(log.close)
+    # printed before any sink starts, so it always comes before the dataset
+    # sink's own "is open" or "is off" line
+    names = ", ".join(s.name + (f" ({s.repo_id}, every {s.every:g} min; opening in "
+                                "the background)"
+                                if isinstance(s, DatasetSink) else f" ({s.path})")
+                      for s in sinks)
+    _warn(f"recording every tool call to: {names}")
+    for sink in sinks:
+        start = getattr(sink, "start", None)
+        if start is not None:
+            start()
+    return log

canlex/charter.py

@@ -0,0 +1,121 @@
+"""Ingest the Charter and border-relevant Constitution Act, 1867 provisions.
+
+The Constitution has no Justice Laws XML, so the corpus long had a hole a
+large fraction of its own case law turns on: Simmons, Canfield, Pike and the
+detention line all apply Charter ss. 7-10 and 24, yet "what does s. 10(b)
+actually say?" returned nothing. Justice Laws publishes the Constitution Acts
+as stable public HTML (laws-lois.justice.gc.ca/eng/const/FullText.html) with
+a regular MarginalNote/Section markup, parsed here into legislation-shaped
+chunks. Scope: the whole Charter (1982 ss. 1-34), Aboriginal rights s. 35,
+the supremacy clause s. 52, and the 1867 heads-of-power ss. 91-92.
+
+    python -m canlex.charter
+"""
+import re
+import sys
+
+from ._common import fetch_cached, norm_ws as _norm, write_corpus
+from .config import PROCESSED_DIR, RAW_DIR
+
+URL = "https://laws-lois.justice.gc.ca/eng/const/FullText.html"
+RAW = RAW_DIR / "charter"
+OUT_FILE = PROCESSED_DIR / "charter.json"
+
+# (act filter, sections wanted). The 1982 Act's Charter is ss. 1-34; s. 35 is
+# Part II (Aboriginal rights) and s. 52 the supremacy clause -- both cited
+# constantly alongside Charter arguments. From 1867, only the ss. 91-92
+# division of powers earns its place in a border-law corpus.
+_WANT_1982 = {str(n) for n in range(1, 36)} | {"52"}
+_WANT_1867 = {"91", "92"}
+
+_MN = re.compile(
+    r'<p class="MarginalNote">(?:<span[^>]*>.*?</span>)?(.*?)</p>', re.S)
+_SEC_ID = re.compile(r'<span class="sectionLabel">([0-9.]+)</span>')
+
+
+def _strip(html):
+    """Readable text from a fragment: block ends become newlines, tags go."""
+    html = re.sub(r"</(?:p|li|h\d)>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    return "\n".join(p for p in (_norm(line) for line in html.split("\n")) if p)
+
+
+def _sections(segment_html):
+    """Yield (number, marginal_note, text) for each section in a segment."""
+    notes = list(_MN.finditer(segment_html))
+    for i, m in enumerate(notes):
+        end = notes[i + 1].start() if i + 1 < len(notes) else len(segment_html)
+        body_html = segment_html[m.end():end]
+        num_m = _SEC_ID.search(body_html)
+        if not num_m:
+            continue
+        yield num_m.group(1), _norm(_strip(m.group(1))), _strip(body_html)
+
+
+def parse(html):
+    """Chunks for every in-scope section of both Acts. Pure, for tests.
+
+    Justice Laws could re-shape this page the way CBSA re-shaped its D-memo
+    index, in which case the MarginalNote/Section markup stops matching and
+    this returns []; the write guard in build() is what keeps that from
+    emptying the corpus.
+    """
+    # The page is both Acts in sequence; split at the 1982 Act's title so
+    # section numbers (which restart) land in the right instrument.
+    cut = html.find("CONSTITUTION ACT, 1982")
+    if cut < 0:
+        raise ValueError("could not locate the Constitution Act, 1982 heading")
+    parts = [
+        ("CONST-1867", "Constitution Act, 1867", "Constitution Act, 1867",
+         html[:cut], _WANT_1867),
+        ("CONST-1982", "Charter", "Constitution Act, 1982 (incl. the Canadian "
+         "Charter of Rights and Freedoms)", html[cut:], _WANT_1982),
+    ]
+    chunks = []
+    for code, short, name, segment, want in parts:
+        for num, note, text in _sections(segment):
+            if num not in want or not text:
+                continue
+            label = f"s. {num}"
+            chunks.append({
+                "id": f"{code}-s{num}",
+                "doc_type": "legislation",
+                "act_code": code,
+                "act_short": short,
+                "act_name": name,
+                "section": num,
+                "marginal_note": note,
+                "part": "",
+                "division": "",
+                "heading": "",
+                "text": text,
+                "history": "",
+                "last_amended": "",
+                "in_force": "",
+                "status": "in force",
+                "current_to": "",
+                "citation": f"{short}, {label}",
+                "source_url": URL,
+            })
+    return chunks
+
+
+def build(allow_shrink=False, force=False):
+    html = fetch_cached(URL, RAW / "FullText.html",
+                        force=force).decode("utf-8", "replace")
+    chunks = parse(html)
+    # indent=2 is what charter.json already carries -- changing it would
+    # rewrite every line of the file for nothing.
+    if not write_corpus(OUT_FILE, chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"charter: {len(chunks)} sections -> {OUT_FILE.name}")
+    return True
+
+
+def main():
+    sys.exit(0 if build(allow_shrink="--allow-shrink" in sys.argv,
+                        force="--force" in sys.argv) else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/citator.py

@@ -0,0 +1,340 @@
+"""CanLII case citator -- live lookup of a Canadian case's citation graph.
+
+Uses the CanLII API (key in canlii_key.txt). The API has no name/topic search,
+so a case is identified by its full canlii.org URL. Responses are cached on disk
+and calls are throttled, because the API rate-limits aggressively.
+
+Build-time only (roadmap T5.5): `python -m canlex.citator --snapshot-authorities`
+records, for each curated authority (data/curated/authorities.json), how many
+decisions cite it on CanLII and the three most recent FC/FCA/SCC ones. It
+reads the key only when the maintainer runs it, and writes
+data/citator_authorities.json, which .gitignore keeps out of the repo and the
+Space. See snapshot_authorities.
+"""
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import date
+from pathlib import Path
+
+from .config import ROOT, DATA_DIR
+
+API = "https://api.canlii.org/v1"
+KEY_FILE = ROOT / "canlii_key.txt"
+_PLACEHOLDER = "PASTE-YOUR-CANLII-API-KEY-ON-THIS-LINE"
+_DBMAP_FILE = DATA_DIR / "citator_dbmap.json"
+_CACHE_FILE = DATA_DIR / "citator_cache.json"
+_THROTTLE = 3.0     # seconds between CanLII API calls (the API rate-limits hard)
+_MAX_LIST = 20      # items shown per citator list (lists can run to thousands)
+_CASE_URL = re.compile(r"canlii\.org/[a-z]{2}/[a-z]+/([a-z0-9-]+)/doc/\d+/([a-z0-9-]+)")
+_DB_URL = re.compile(r"canlii\.org/[a-z]{2}/[a-z]+/([a-z0-9-]+)/?$")
+
+# A neutral citation, e.g. "2019 SCC 65", and the CanLII URL segment per
+# court/tribunal. The tribunal entries cover the boards whose decisions the
+# corpus itself carries, so the natural loop (search result -> canlex_case)
+# works for every citation a result can display.
+# 'FCT' is the Federal Court's neutral citation for 2001-2003 ('2002 FCT
+# 123'); CanLII files those decisions under 'fct' with that document id.
+_NEUTRAL = re.compile(
+    r"\b(\d{4})\s+(SCC|FCA|FCT|FC|FPSLREB|CIRB|ONCA|ABCA)\s+(\d+)\b",
+    re.IGNORECASE)
+_CANLII_SEG = {"scc": "scc", "fca": "fca", "fc": "fct", "fct": "fct",
+               "fpslreb": "pslreb", "cirb": "cirb",
+               "onca": "onca", "abca": "abca"}
+_PROVINCE_SEG = {"onca": "on", "abca": "ab"}  # others are federal ('ca')
+
+
+def canlii_url_from_citation(text):
+    """Build a canlii.org case URL from a neutral citation, or '' if none found.
+
+    Works for the Supreme Court, Federal Court of Appeal, Federal Court, the
+    FPSLREB and CIRB labour boards, and the Ontario and Alberta Courts of
+    Appeal -- e.g. "2019 SCC 65" -> .../en/ca/scc/doc/2019/2019scc65/...
+    """
+    m = _NEUTRAL.search(text)
+    if not m:
+        return ""
+    year, court, num = m.group(1), m.group(2).lower(), m.group(3)
+    doc = f"{year}{court}{num}"
+    juris = _PROVINCE_SEG.get(court, "ca")
+    return (f"https://www.canlii.org/en/{juris}/{_CANLII_SEG[court]}/doc/"
+            f"{year}/{doc}/{doc}.html")
+
+
+def api_key():
+    """Return the configured CanLII API key, or '' if not set.
+
+    The CANLII_API_KEY environment variable is checked first -- the remote
+    deployment injects the key as a secret rather than shipping the file.
+    A local run falls back to canlii_key.txt.
+    """
+    env_key = os.environ.get("CANLII_API_KEY", "").strip()
+    if env_key and env_key != _PLACEHOLDER:
+        return env_key
+    if not KEY_FILE.exists():
+        return ""
+    key = KEY_FILE.read_text(encoding="utf-8").strip()
+    return "" if not key or key == _PLACEHOLDER else key
+
+
+def _load_json(path):
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+
+
+class Citator:
+    """Live CanLII citator with on-disk caching and rate-limit throttling."""
+
+    def __init__(self):
+        self.key = api_key()
+        if not self.key:
+            raise RuntimeError(f"No CanLII API key -- put your key in {KEY_FILE}.")
+        self._dbmap = _load_json(_DBMAP_FILE)   # URL court segment -> databaseId
+        self._cache = _load_json(_CACHE_FILE)   # case URL -> report
+
+    def _get(self, path):
+        """Throttled GET against the CanLII API, retrying on HTTP 429."""
+        for attempt in range(3):
+            time.sleep(_THROTTLE if attempt == 0 else 15.0)
+            sep = "&" if "?" in path else "?"
+            url = f"{API}/{path}{sep}api_key={self.key}"
+            try:
+                with urllib.request.urlopen(url, timeout=45) as resp:
+                    return json.loads(resp.read().decode("utf-8"))
+            except urllib.error.HTTPError as exc:
+                if exc.code == 429 and attempt < 2:
+                    continue
+                raise
+        raise RuntimeError("CanLII API rate limit reached; retry shortly.")
+
+    def _ensure_dbmap(self):
+        if self._dbmap:
+            return
+        data = self._get("caseBrowse/en/")
+        dbmap = {}
+        for db in data.get("caseDatabases", []):
+            match = _DB_URL.search(db.get("url") or "")
+            if match and db.get("databaseId"):
+                dbmap[match.group(1)] = db["databaseId"]
+        self._dbmap = dbmap
+        DATA_DIR.mkdir(parents=True, exist_ok=True)
+        _DBMAP_FILE.write_text(json.dumps(dbmap), encoding="utf-8")
+
+    def _citator(self, db, case_id, kind):
+        data = self._get(f"caseCitator/en/{db}/{case_id}/{kind}")
+        items = data.get(kind, [])
+        return {"total": len(items), "items": items[:_MAX_LIST]}
+
+    def case_report(self, case_url):
+        """Return a citation-graph report for a case.
+
+        Accepts a full canlii.org case URL, or a neutral citation (e.g.
+        "2019 SCC 65") for a Supreme Court / Federal Court of Appeal / Federal
+        Court decision.
+        """
+        if not _CASE_URL.search(case_url):
+            case_url = canlii_url_from_citation(case_url) or case_url
+        if case_url in self._cache:
+            return self._cache[case_url]
+        match = _CASE_URL.search(case_url)
+        if not match:
+            return {"error": "Provide a full canlii.org case URL, or a neutral "
+                    "citation such as '2019 SCC 65' (Supreme Court, Federal "
+                    "Court of Appeal, or Federal Court)."}
+        self._ensure_dbmap()
+        segment, case_id = match.group(1), match.group(2)
+        db = self._dbmap.get(segment)
+        if not db:
+            return {"error": f"Unrecognized CanLII court segment '{segment}'."}
+        report = {
+            "meta": self._get(f"caseBrowse/en/{db}/{case_id}/"),
+            "citingCases": self._citator(db, case_id, "citingCases"),
+            "citedCases": self._citator(db, case_id, "citedCases"),
+            "citedLegislations": self._citator(db, case_id, "citedLegislations"),
+        }
+        self._cache[case_url] = report
+        DATA_DIR.mkdir(parents=True, exist_ok=True)
+        _CACHE_FILE.write_text(json.dumps(self._cache, ensure_ascii=False),
+                               encoding="utf-8")
+        return report
+
+
+    def citing_cases(self, case_url):
+        """Every CanLII citingCases item for a canlii.org case URL, uncapped
+        (case_report keeps _MAX_LIST), or None when the URL is not a case
+        URL or its court is not in the database map. Not cached."""
+        match = _CASE_URL.search(case_url or "")
+        if not match:
+            return None
+        self._ensure_dbmap()
+        db = self._dbmap.get(match.group(1))
+        if not db:
+            return None
+        data = self._get(f"caseCitator/en/{db}/{match.group(2)}/citingCases")
+        return data.get("citingCases", [])
+
+
+# --- Build-time authority snapshot (roadmap T5.5) ----------------------------
+# canlex_case needs a key the Space may not have. So the maintainer runs,
+# locally and with his own key,
+#     python -m canlex.citator --snapshot-authorities
+# which records, for every curated authority with a CanLII case URL, how
+# many decisions cite it on CanLII and the three most recent FC/FCA/SCC
+# ones. The file is data/citator_authorities.json, which .gitignore keeps
+# out of the repo and the Space: whether derived CanLII API data may ship
+# is the maintainer's open question. A citation count is not a treatment
+# check -- it cannot say whether a later court followed, distinguished or
+# reversed the authority, and it would not have caught the reversal of the
+# trial-level Saini decision -- so the registry's manual subsequent_history
+# stays the only 'reversed' signal.
+SNAPSHOT_FILE = DATA_DIR / "citator_authorities.json"
+SNAPSHOT_AGING_DAYS = 180
+_APEX_COURTS = ("SCC", "FCA", "FC", "FCT")
+_REGISTRY_FILE = DATA_DIR / "curated" / "authorities.json"
+
+
+def authority_case_url(row):
+    """The CanLII case URL for an authority-registry row: its source_url
+    when that is a canlii.org case URL, else one built from a neutral
+    citation in its citation, decision key or cites; '' when it has none
+    (pre-2001 reports held on Lexum, secondary-only rows)."""
+    url = row.get("source_url") or ""
+    if _CASE_URL.search(url):
+        return url
+    texts = [row.get("citation"), (row.get("decision_key") or {}).get("act_code")]
+    for text in texts + list(row.get("cites") or []):
+        built = canlii_url_from_citation(text or "")
+        if built:
+            return built
+    return ""
+
+
+def _neutral_order(citation):
+    m = _NEUTRAL.search(citation or "")
+    return (int(m.group(1)), int(m.group(3))) if m else (0, 0)
+
+
+def apex_citing(items):
+    """The FC, FCA and SCC decisions among CanLII citingCases items, by
+    their neutral citation, most recent first (year, then number)."""
+    out = []
+    for item in items or []:
+        cite = item.get("citation") or ""
+        m = _NEUTRAL.search(cite)
+        if m and m.group(2).upper() in _APEX_COURTS:
+            out.append({"title": item.get("title", ""), "citation": cite})
+    out.sort(key=lambda r: _neutral_order(r["citation"]), reverse=True)
+    return out
+
+
+def snapshot_authorities(citer, registry=None, today=None, path=SNAPSHOT_FILE,
+                         write=True):
+    """Build the authority snapshot with `citer` (a Citator, or anything with
+    citing_cases(url)) and write it to `path` unless write=False. Each row
+    that resolves: {url, as_of, citing_total, citing_apex, recent_apex (the
+    three most recent FC/FCA/SCC citing decisions)}. A row with no CanLII
+    URL, or whose lookup fails, lands in 'skipped' with the reason -- an
+    error by its type and HTTP code only, never its text, because the
+    request URL carries the API key. A run in which no row resolved (every
+    lookup failed: a rate limit, the network) is not written, so it never
+    overwrites a good earlier snapshot with an all-skipped one."""
+    if registry is None:
+        registry = json.loads(_REGISTRY_FILE.read_text(encoding="utf-8"))
+    today = today or date.today().isoformat()
+    out = {"as_of": today,
+           "source": "CanLII API caseCitator citingCases (python -m canlex.citator "
+                     "--snapshot-authorities)",
+           "note": "a citation count, not a treatment check; the registry's "
+                   "subsequent_history is the only 'reversed' signal",
+           "authorities": {}, "skipped": {}}
+    for key, row in sorted((registry.get("authorities") or {}).items()):
+        url = authority_case_url(row)
+        if not url:
+            out["skipped"][key] = ("no CanLII case URL (no canlii.org source_url "
+                                   "and no neutral citation)")
+            continue
+        try:
+            items = citer.citing_cases(url)
+        except Exception as exc:        # network or API failure: keep going
+            code = getattr(exc, "code", "")
+            out["skipped"][key] = f"lookup failed: {type(exc).__name__} {code}".strip()
+            continue
+        if items is None:
+            out["skipped"][key] = "court not in the CanLII database map"
+            continue
+        apex = apex_citing(items)
+        out["authorities"][key] = {"url": url, "as_of": today,
+                                   "citing_total": len(items),
+                                   "citing_apex": len(apex),
+                                   "recent_apex": apex[:3]}
+    if write and out["authorities"]:
+        Path(path).write_text(json.dumps(out, indent=1, ensure_ascii=False),
+                              encoding="utf-8")
+    return out
+
+
+def load_snapshot(path=SNAPSHOT_FILE):
+    """The snapshot, or None where it was never built (every deployment:
+    the file is gitignored)."""
+    try:
+        return json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def snapshot_line(entry, today=None):
+    """One authority's snapshot as the officer-facing line:
+    'Cited by N later FC/FCA/SCC decisions on CanLII as of <date>; this is
+    not a treatment check', the most recent of them, and AGING once the
+    snapshot is older than SNAPSHOT_AGING_DAYS."""
+    line = (f"Cited by {entry['citing_apex']} later FC/FCA/SCC decisions on "
+            f"CanLII as of {entry['as_of']}; this is not a treatment check")
+    recent = "; ".join(r["citation"] for r in entry.get("recent_apex") or [])
+    if recent:
+        line += f" (most recent: {recent})"
+    age = (date.fromisoformat(today or date.today().isoformat())
+           - date.fromisoformat(entry["as_of"])).days
+    if age > SNAPSHOT_AGING_DAYS:
+        line += (f" -- AGING: the snapshot is {age} days old; re-run "
+                 f"python -m canlex.citator --snapshot-authorities")
+    return line + "."
+
+
+def main():
+    if sys.argv[1:2] == ["--snapshot-authorities"]:
+        snap = snapshot_authorities(Citator())
+        for key, entry in snap["authorities"].items():
+            print(f"{key}: {snapshot_line(entry)}")
+        for key, why in snap["skipped"].items():
+            print(f"{key}: skipped -- {why}")
+        if snap["authorities"]:
+            print(f"-> {SNAPSHOT_FILE} (gitignored; not shipped)")
+        else:
+            print(f"no authority resolved -- nothing written; {SNAPSHOT_FILE} "
+                  f"(if any) is kept")
+        return
+    if len(sys.argv) < 2:
+        print('usage: python -m canlex.citator "<canlii-case-url>"\n'
+              '       python -m canlex.citator --snapshot-authorities')
+        return
+    report = Citator().case_report(sys.argv[1])
+    if "error" in report:
+        print("ERROR:", report["error"])
+        return
+    meta = report["meta"]
+    print(f"{meta.get('title')} -- {meta.get('citation')}  ({meta.get('decisionDate')})")
+    print(f"  cited by:  {report['citingCases']['total']}")
+    print(f"  cites:     {report['citedCases']['total']}")
+    print(f"  legislation cited: {report['citedLegislations']['total']}")
+    for item in report["citedLegislations"]["items"][:6]:
+        print(f"    - {item.get('title')} ({item.get('citation')})")
+
+
+if __name__ == "__main__":
+    main()

canlex/ciu.py

@@ -0,0 +1,292 @@
+"""Ingest CIU and PSAC union documents (doc_type='union').
+
+The Customs and Immigration Union is the PSAC component representing the FB
+bargaining unit; its governance instruments and published reference material
+are the union-side counterpart to the FB collective agreement already in the
+corpus. Ingested here, all as public documents from ciu-sdi.ca / psacunion.ca:
+
+- the CIU National By-Laws (EN + FR), CIU Regulations, Branch Rules and the
+  Local Modifications to Branch Rules -- the union's internal law;
+- the PSAC Constitution and Regulations (the parent instrument the by-laws
+  operate under: dues, discipline and appeals bottom out there);
+- the Branch Elections Kit and the members' guides (grievance handling,
+  employment equity, workers in transition);
+- every site-specific Variable Shift Schedule Agreement (VSSA) the union
+  publishes -- the locally negotiated schedule grids for ports of entry.
+
+None of this is employer policy or part of the collective agreement itself;
+_format_section banners doc_type='union' accordingly.
+
+    python -m canlex.ciu [--force] [--allow-shrink]
+
+--force re-downloads every cached PDF instead of reusing data/raw/ciu.
+The union reposts amended documents at the same URLs, so a plain run can
+never notice an upstream revision -- see canlex.refresh.check_union.
+--allow-shrink lets a run that yields fewer than 80% of the stored chunks
+write (the corpus-write guard, _common.write_corpus); a document or VSSA
+that fails to fetch keeps its last-good chunks (_common.preserve_dropped).
+"""
+import re
+import sys
+import urllib.request
+
+from ._common import (dehyphenate, document_words, fetch_cached,
+                      mark_soft_breaks, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "ciu"
+VSSA_PAGE = "https://ciu-sdi.ca/en/vssa/"
+_DOCS_BASE = "https://www.ciu-sdi.ca/wp-content/uploads/documents/"
+
+# (code, act_short, act_name, url, lang)
+DOCS = [
+    ("ciu-bylaws", "CIU By-Laws",
+     "CIU National By-Laws (adopted by CIU National Conventions)",
+     _DOCS_BASE + "bylaws_en.pdf", "en"),
+    ("ciu-bylaws", "CIU By-Laws",
+     "Règlements nationaux du SDI (adoptés par les congrès)",
+     _DOCS_BASE + "bylaws_fr.pdf", "fr"),
+    ("ciu-regs", "CIU Regulations",
+     "CIU Regulations (adopted by the CIU National Board of Directors)",
+     _DOCS_BASE + "ciu-regs-en.pdf", "en"),
+    ("ciu-branch-rules", "CIU Branch Rules",
+     "CIU Branch Rules (adopted by the CIU National Board of Directors)",
+     _DOCS_BASE + "branch-rules-en.pdf", "en"),
+    ("ciu-local-mods", "CIU Local Modifications",
+     "Local Modifications to CIU Branch Rules",
+     _DOCS_BASE + "local_mod_branch_rules_en.pdf", "en"),
+    ("ciu-elections-kit", "CIU Elections Kit",
+     "CIU Branch Elections Kit",
+     "https://www.ciu-sdi.ca/wp-content/uploads/elections/2023/EN/"
+     "elections-kit.pdf", "en"),
+    ("psac-constitution", "PSAC Constitution",
+     "PSAC Constitution and Regulations (November 2024 edition)",
+     "https://psacunion.ca/sites/psac/files/"
+     "psac.constitution.november_2024_edition.eng_v002.pdf", "en"),
+    ("ciu-guide-grievance", "CIU Grievance Guide",
+     "CIU Guide to Grievance Handling",
+     "https://ciu-sdi.ca/wp-content/uploads/2012/09/"
+     "guide_to_grievance_handling-e.pdf", "en"),
+    ("ciu-guide-ee", "CIU EE Toolkit",
+     "CIU Employment Equity Toolkit",
+     "https://ciu-sdi.ca/wp-content/uploads/2012/09/"
+     "EMPLOYMENT-EQUITY-Tool-Kit-e.pdf", "en"),
+    ("ciu-guide-transition", "CIU Transition Guide",
+     "Workers in Transition: a practical guide about gender transition",
+     "https://ciu-sdi.ca/wp-content/uploads/2012/09/"
+     "workers_in_transition_guide_-_en_-_web.pdf", "en"),
+]
+
+# Governance headings: "BY-LAW 5", "REGULATION 12", "RULE 3", "ARTICLE 7",
+# "SECTION 25", "PART B" -- at line start, optionally followed by a title.
+_HEADING = re.compile(
+    r"^(BY-?LAW|R[EÈ]GLEMENT|REGULATION|RULE|ARTICLE|SECTION|PART|ANNEX(?:E)?"
+    r"|APPENDIX)\s+([0-9]+[0-9A-Za-z.\-]*|[A-Z])\b\s*[-–—:]?\s*(.{0,80})",
+    re.IGNORECASE)
+_DATE = re.compile(r"\b(20\d{2}|19\d{2})\b")
+
+
+def _pdf_pages(path):
+    from pypdf import PdfReader
+    pages = []
+    for page in PdfReader(str(path)).pages:
+        try:
+            pages.append(page.extract_text() or "")
+        except Exception:
+            pages.append("")
+    # a line-final soft hyphen is a broken word's printed hyphen: kept as
+    # '-' so it survives norm_ws for dehyphenate (2026-09-22)
+    return [mark_soft_breaks(page) for page in pages]
+
+
+def _currency(pages):
+    """Newest adoption/amendment year on the cover pages."""
+    years = _DATE.findall("\n".join(pages[:3]))
+    return max(years) if years else ""
+
+
+def _doc_chunks(code, short, name, url, lang, pages):
+    text = "\n".join(pages)
+    lines = text.split("\n")
+    currency = _currency(pages)
+    marks = []
+    for i, l in enumerate(lines):
+        m = _HEADING.match(l.strip())
+        if m:
+            marks.append((i, f"{m.group(1).title()} {m.group(2)}",
+                          _norm(m.group(3)) or ""))
+    segments = []
+    if len(marks) >= 6:
+        for j, (i, label, title) in enumerate(marks):
+            end = marks[j + 1][0] if j + 1 < len(marks) else len(lines)
+            body = "\n".join(p for p in (_norm(x) for x in lines[i:end]) if p)
+            if len(body) > 120:
+                segments.append((label, title, body))
+    if not segments:
+        for k, piece in enumerate(split_lines(text, 1800), start=1):
+            body = "\n".join(p for p in (_norm(x) for x in piece.split("\n")) if p)
+            if len(body) > 120:
+                segments.append((f"part {k}", "", body))
+    chunks = []
+    fr = "-fr" if lang == "fr" else ""
+    words = document_words(lines)
+    for n, (label, title, body) in enumerate(segments, start=1):
+        body = dehyphenate(body, words)      # line-break hyphens (2026-09-22)
+        pieces = split_lines(body, 1800) if len(body) > 2400 else [body]
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            chunk = {
+                "id": f"{code}{fr}-{n}{suffix}",
+                "doc_type": "union",
+                "act_code": code.upper(),
+                "act_short": short,
+                "act_name": name,
+                "section": label,
+                "marginal_note": title,
+                "part": short,
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": currency,
+                "in_force": "",
+                "status": "in force",
+                "current_to": currency,
+                "citation": f"{short}, {label}"
+                            + (f" ({title})" if title else "") + note,
+                "source_url": url,
+            }
+            if lang == "fr":
+                chunk["lang"] = "fr"
+            chunks.append(chunk)
+    return chunks
+
+
+def _vssa_links():
+    req = urllib.request.Request(VSSA_PAGE, headers={"User-Agent": "Mozilla/5.0"})
+    with urllib.request.urlopen(req, timeout=60) as r:
+        html = r.read().decode("utf-8", "replace")
+    links = sorted(set(re.findall(
+        r'href="(https?://[^"]*?/uploads/vssa/([^"/]+)\.pdf)"', html)))
+    return links
+
+
+def _vssa_chunks(url, stem, pages):
+    site = _norm(stem.replace("-", " "))
+    text = "\n".join(p for p in (_norm(x) for x in "\n".join(pages).split("\n")) if p)
+    text = dehyphenate(text, document_words(pages))   # line-break hyphens
+    if len(text) < 60:
+        return []
+    out = []
+    pieces = split_lines(text, 1800)
+    total = len(pieces)
+    for k, piece in enumerate(pieces, start=1):
+        suffix = "" if total == 1 else f"-p{k}"
+        note = "" if total == 1 else f" (part {k} of {total})"
+        out.append({
+            "id": f"vssa-{stem.lower()}{suffix}",
+            "doc_type": "union",
+            "act_code": "VSSA",
+            "act_short": "VSSA",
+            "act_name": "Variable Shift Schedule Agreements (by work site)",
+            "section": site,
+            "marginal_note": f"VSSA — {site}",
+            "part": "Variable Shift Schedule Agreements",
+            "division": "",
+            "heading": "",
+            "text": piece,
+            "history": "",
+            "last_amended": "",
+            "in_force": "",
+            "status": "in force",
+            "current_to": "",
+            "citation": f"VSSA — {site}{note}",
+            "source_url": url,
+        })
+    return out
+
+
+def _identity(chunk):
+    """The document a chunk belongs to, for preserve-on-failure: a governance
+    document is (ACT_CODE, lang, ''); a VSSA is ('VSSA', 'en', site), since
+    every site shares the VSSA act_code and is told apart by its section."""
+    code = chunk["act_code"]
+    site = chunk["section"] if code == "VSSA" else ""
+    return (code, chunk.get("lang", "en"), site)
+
+
+def build(force=False, allow_shrink=False):
+    """Rebuild ciu.json. force=True re-downloads each PDF; without it the
+    cached copy in data/raw/ciu wins and upstream revisions are invisible.
+    Returns the chunks, or False when the corpus-write guard refused.
+
+    A document or VSSA that fails to fetch or parse keeps its last-good
+    chunks (restricted to the ones that actually failed, so a VSSA the union
+    takes down stays gone once the index no longer lists it). Until
+    2026-09-21 this ingester wrote ciu.json directly: a run with the union
+    site down wrote an empty list over 635 stored chunks and exited 0."""
+    RAW.mkdir(parents=True, exist_ok=True)
+    out = PROCESSED_DIR / "ciu.json"
+    stored = stored_chunks(out)
+    all_chunks, failed = [], set()
+    for code, short, name, url, lang in DOCS:
+        fr = "-fr" if lang == "fr" else ""
+        dest = RAW / f"{code}{fr}.pdf"
+        try:
+            fetch_cached(url, dest, pause=0.8, force=force)
+            chunks = _doc_chunks(code, short, name, url, lang,
+                                 _pdf_pages(dest))
+        except Exception as exc:
+            print(f"  !! {short} ({lang}): {type(exc).__name__}: {exc}")
+            failed.add((code.upper(), lang, ""))
+            continue
+        print(f"  {short:24} ({lang}): {len(chunks)} chunks")
+        all_chunks.extend(chunks)
+    try:
+        links = _vssa_links()
+    except Exception as exc:
+        links = []
+        print(f"  !! VSSA index: {type(exc).__name__}: {exc}")
+        # No index at all says nothing about any one site: keep every stored VSSA.
+        failed.update(_identity(c) for c in stored if c.get("act_code") == "VSSA")
+    n_vssa = 0
+    for url, stem in links:
+        dest = RAW / f"vssa-{stem.lower()}.pdf"
+        try:
+            fetch_cached(url, dest, pause=0.6, force=force)
+            chunks = _vssa_chunks(url, stem, _pdf_pages(dest))
+        except Exception as exc:
+            print(f"  !! VSSA {stem}: {type(exc).__name__}: {exc}")
+            failed.add(("VSSA", "en", _norm(stem.replace("-", " "))))
+            continue
+        all_chunks.extend(chunks)
+        n_vssa += 1
+    print(f"  VSSA sites: {n_vssa}")
+    all_chunks, preserved = preserve_dropped(all_chunks, stored, key=_identity,
+                                             only=failed)
+    if preserved:
+        names = ", ".join(f"{code}{' (' + lang + ')' if lang != 'en' else ''}"
+                          f"{' ' + site if site else ''}"
+                          for code, lang, site in preserved)
+        print(f"  kept the last-good copy of {len(preserved)} unfetchable "
+              f"document(s): {names}")
+    uniquify_ids(all_chunks)
+    if not write_corpus(out, all_chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"ciu: {len(all_chunks)} chunks -> {out.name}")
+    return all_chunks
+
+
+def main():
+    # `is not False` rather than truthiness: a first-ever run that writes an
+    # empty corpus succeeded, a refused write did not.
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv) is not False
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/commentary.py

@@ -0,0 +1,482 @@
+"""Ingest CanLex's curated commentary datasets into searchable chunks.
+
+Commentary is the one doc_type CanLex AUTHORS rather than mirrors: structured
+legal-analysis datasets (currently the US-dispositions helper -- whether each
+kind of US state criminal disposition is a "conviction" for IRPA s. 36
+purposes). Every chunk is banner-labelled as commentary, every proposition
+carries its authorities, and entries with no authority say so explicitly and
+flag their reasoning as interpretation. The source of truth is
+data/curated/us_dispositions.json, which is reviewed by the user before it
+ships; this module just renders it into corpus chunks.
+
+    py -m canlex.commentary [--allow-shrink]
+"""
+import json
+import re
+import sys
+
+from ._common import write_corpus
+from .config import DATA_DIR, PROCESSED_DIR
+
+CURATED = DATA_DIR / "curated" / "us_dispositions.json"
+EQUIV = DATA_DIR / "curated" / "us_equivalency.json"
+OUT = PROCESSED_DIR / "commentary.json"
+
+ACT_CODE = "US-DISP"
+ACT_SHORT = "US Dispositions Helper"
+ACT_NAME = ("US criminal dispositions and the IRPA 'conviction' concept "
+            "(curated CanLex commentary)")
+
+BANNER = ("CURATED ANALYSIS -- commentary compiled for CanLex, not a source "
+          "of law. Verify against the cited authorities before relying on it.")
+
+# The heading over an entry's guidance references (roadmap T1.10). It used to
+# say 'IRCC guidance (cited by reference; not reproduced here)', but the
+# references are mostly IRB Legal Services papers, and the guidance CanLex
+# holds is now in the corpus. canlex/server.py's _entry_brief shares it.
+GUIDANCE_HEADING = ("Guidance cited (IRB / IRCC; where CanLex holds the text, "
+                    "search it with canlex_search_legislation, doc_type "
+                    "'memorandum'):")
+
+_STATUS_LABEL = {
+    "settled": "Settled by binding authority",
+    "judicially-considered": "Judicially considered (persuasive authority)",
+    "guidance-only": "IRCC guidance only -- no judicial authority located",
+    "no-authority": "NO AUTHORITY LOCATED -- reasoned interpretation only",
+}
+
+# Verdicts assess the COMPLETED disposition; the methodology chunk explains
+# the rule each one derives from. 'depends' is the legacy spelling of
+# fact-specific, kept for backwards compatibility.
+_VERDICT_LABEL = {
+    "yes": "YES", "likely-yes": "LIKELY YES", "likely-no": "LIKELY NO",
+    "no": "NO", "fact-specific": "FACT-SPECIFIC",
+    "depends": "FACT-SPECIFIC",
+}
+
+
+def _vlabel(v):
+    return _VERDICT_LABEL.get(v, str(v).upper())
+
+
+# The survey's own caution, rendered with every entry that carries state
+# rows: most rows are secondary-source extractions and the dataset says so
+# in its `_state_survey` key -- which nothing used to read.
+SURVEY_NOTE = ("State rows: 51-jurisdiction survey (July 2026); verdicts applied "
+               "by rule from each row's mechanism facts. Every row shows its "
+               "provenance: a row marked 'verified <date> against <source>' was "
+               "checked against the official text it names (verification rounds "
+               "2026-07-12 to 2026-09-12); a row marked 'NOT VERIFIED' says why; "
+               "every other row is a secondary-source extraction -- confirm the "
+               "statute before relying on it to clear a record.")
+
+# Compiler-side notes the survey left in row text ("[EXTRACTION FLAG: ...
+# verify]", "PROFILE THIN -- VERIFY"): rendered as explicit open questions,
+# never as analysis.
+_FLAG = re.compile(r"\s*\[(?:EXTRACTION FLAG|Corrected|Normalized)[^\]]*\]")
+# A sentence boundary: a period followed by whitespace and a capital -- so
+# 'art. 42A.701' and 'Stat. Ann. § 431.073' stay whole.
+_SENTENCE = re.compile(r"(?<=\.)\s+(?=[A-Z])")
+_VERIFY = re.compile(r"\bVERIFY\b|\bPROFILE (?:THIN|SILENT)\b")
+
+
+def _split_verify(note):
+    """(note without its compiler notes, [the notes]): whole sentences that
+    carry a VERIFY / PROFILE THIN marker are lifted out of the row text."""
+    keep, lifted = [], []
+    for sentence in _SENTENCE.split(note):
+        (lifted if _VERIFY.search(sentence) else keep).append(sentence.strip())
+    return " ".join(s for s in keep if s), lifted
+
+
+def _sources(v):
+    """A state row's 'source' (roadmap T3.5, held): one object or a list."""
+    s = v.get("source")
+    if isinstance(s, dict):
+        return [s]
+    return [x for x in (s or []) if isinstance(x, dict)]
+
+
+def provenance_label(v):
+    """The provenance tier printed with EVERY state row (roadmap T3.5): where
+    its statement of state law comes from. A verified row names the official
+    text it was checked against; a row the verification could not reach says
+    why; any other row is a secondary-source extraction."""
+    src = _sources(v)
+    if v.get("verified"):
+        if src:
+            s = src[0]
+            grade = f", {s['grade']}" if s.get("grade") else ""
+            return f"[verified {v['verified']} against {s['cite']}{grade}: {s['url']}]"
+        return f"[verified against statute text {v['verified'][:7]}]"
+    if v.get("unverifiable"):
+        read = f"; read: {src[0]['cite']}: {src[0]['url']}" if src else ""
+        return f"[NOT VERIFIED: {v['unverifiable']}{read}]"
+    return "[secondary-source extraction, unverified]"
+
+
+def row_tier(v):
+    """The same tier in a few words, for the screening memo's step 1."""
+    src = _sources(v)
+    if v.get("verified"):
+        return (f"verified {v['verified']} against {src[0]['cite']}" if src
+                else "verified against statute text")
+    if v.get("unverifiable"):
+        return f"NOT VERIFIED -- {v['unverifiable']}"
+    return "secondary-source extraction -- unverified"
+
+
+def row_tag(v):
+    """The tier as a short tag for a one-line index of rows (wave-3 verifier,
+    2026-09-15: the 'all' index listed every row with its verdict only)."""
+    if v.get("verified"):
+        return f"[verified {v['verified']}]"
+    if v.get("unverifiable"):
+        return "[NOT VERIFIED]"
+    return "[unverified]"
+
+
+def row_notes(v):
+    """The held review lines a state row carries (roadmap T3.5): a CanLex
+    characterisation of the state's law, and what decides a split row."""
+    out = []
+    if v.get("characterisation"):
+        out.append("CANLEX CHARACTERISATION (held for review): " + v["characterisation"])
+    if v.get("open_question"):
+        out.append("WHAT DECIDES IT (held for review): " + v["open_question"])
+    return out
+
+
+def row_provenance(v):
+    """CANLEX_RESULT 'row_provenance' for a state row; None without a row."""
+    if not v:
+        return None
+    s = (_sources(v) or [{}])[0]
+    return {"state": v.get("state"),
+            "tier": ("verified" if v.get("verified") else
+                     "not-verified" if v.get("unverifiable") else "secondary-unverified"),
+            "verified": v.get("verified"), "verified_by": v.get("verified_by"),
+            "source_cite": s.get("cite"), "source_url": s.get("url"),
+            "unverifiable": v.get("unverifiable"),
+            "characterisation": bool(v.get("characterisation")),
+            "open_question": bool(v.get("open_question"))}
+
+
+def _row_lines(v, verdict_label, full=False):
+    """The bullet for one state row with its provenance tier (every row,
+    roadmap T3.5), the held review lines, any open-question line, and --
+    full=True, the requested state in a tool answer -- each source quote."""
+    note = v["note"]
+    flags = [m.group(0).strip(" []") for m in _FLAG.finditer(note)]
+    note = _FLAG.sub("", note)
+    note, verify = _split_verify(note)
+    lines = [f"- {v['state']}{verdict_label}: {note} {provenance_label(v)}"]
+    lines += ["  " + x for x in row_notes(v)]
+    if full:
+        for s in _sources(v):
+            if s.get("quote"):
+                lines.append(f"  SOURCE: {s['cite']}: \"{s['quote']}\" ({s['url']}, "
+                             f"retrieved {s.get('retrieved', '?')})")
+    open_q = [f for f in flags if not f.startswith("Normalized")] + verify
+    if open_q:
+        lines.append("  OPEN QUESTION (unverified extraction note): "
+                     + " | ".join(open_q))
+    return lines
+
+
+# A disposition entry may carry 'policy_caveats' (roadmap T3.6, 2026-09-15 --
+# HELD FOR REVIEW; optional, the approved baseline has none): official guidance
+# that pulls against the entry's bottom line, quoted verbatim (each quote pinned
+# in the corpus by tests/test_read_next.py), with both readings -- principle 5:
+# CanLex does not choose. First use: ENF 2 s. 3's list of scenarios in which the
+# act branch 'would generally not be applied' against the entries whose bottom
+# lines keep it available.
+POLICY_CAVEAT_HEAD = "POLICY CAVEAT -- NOT SETTLED (both readings are given; neither is chosen):"
+# Disposition roadmap T1.5 / T1.10 and the critic's Rodriguez Anzola item
+# (2026-09-16, HELD): what a caveat bears on, keyed by its curated id -- never
+# by its text. 'act': the act branch, whose allegation is still drafted with
+# the caveat beside it (ENF 2 s. 3's scenarios); 'act-not-settled': the act
+# branch after an acquittal, where no act-branch allegation, s. 44(1) skeleton
+# or card report is drafted (ENF 6 s. 23.1 against the statute and Magtibay);
+# 'act-if-acquitted': a pending charge, whose readings apply only if it ends
+# in an acquittal; 'conviction': the conviction branch itself (a conviction
+# that may be a nullity for Canadian purposes). An id not listed is 'act'.
+CAVEAT_CLASSES = {
+    "enf-2-s3-act-branch": "act",
+    "enf-6-s23-1-acquittal": "act-not-settled",
+    "enf-6-s23-1-if-acquitted": "act-if-acquitted",
+    "rodriguez-anzola-duress-nullity": "conviction",
+}
+AUTHORITY_CAVEAT_HEAD = ("AUTHORITY CAVEAT -- NOT SETTLED (both readings are given; neither is "
+                         "chosen):")
+
+
+def caveat_class(cv):
+    """The class CAVEAT_CLASSES gives a curated caveat item ('act' by default)."""
+    return CAVEAT_CLASSES.get((cv or {}).get("id") if isinstance(cv, dict) else None, "act")
+
+
+def caveat_in_play(cv, footing):
+    """True when a caveat bears on a step-1 footing ('conviction', 'act' or
+    'both'; None: every caveat, as the entry prints without a record)."""
+    if footing is None or footing == "both":
+        return True
+    return (caveat_class(cv) == "conviction") == (footing == "conviction")
+
+
+def policy_caveat_lines(e, footing=None, skip=()):
+    """The POLICY CAVEAT lines of an entry: the caveat text with its source and
+    chunk, then each reading ([] when the entry carries none). footing: only
+    the caveats that bear on it (caveat_in_play); skip: caveat ids the caller
+    prints another way."""
+    out = []
+    for cv in (e.get("policy_caveats") or []) if isinstance(e, dict) else []:
+        if not (isinstance(cv, dict) and isinstance(cv.get("text"), str) and cv["text"].strip()):
+            continue
+        if cv.get("id") in skip or not caveat_in_play(cv, footing):
+            continue
+        head = AUTHORITY_CAVEAT_HEAD if caveat_class(cv) == "conviction" else POLICY_CAVEAT_HEAD
+        where = "; ".join(x for x in (cv.get("source"), cv.get("chunk")) if x)
+        out.append(f"{head} {cv['text']}" + (f" [{where}]" if where else ""))
+        out += [f"- {r}" for r in cv.get("readings") or [] if isinstance(r, str) and r.strip()]
+    return out
+
+
+def _entry_text(e, max_states=None):
+    """Render one disposition entry as a readable, retrieval-friendly block.
+
+    With the 51-jurisdiction survey an entry can carry dozens of state rows;
+    max_states caps how many render inline (the corpus keeps full per-state
+    detail in separate per-state chunks so retrieval still reaches every
+    row). None renders everything -- the tool uses that for direct lookups."""
+    lines = [BANNER, ""]
+    lines.append(f"Disposition: {'; '.join(e['names'])}")
+    lines.append(f"Is the COMPLETED disposition a conviction for IRPA s. 36: "
+                 f"{_vlabel(e['is_conviction'])}")
+    lines.append(f"Authority status: {_STATUS_LABEL[e['status']]}")
+    if e.get("bottom_line"):
+        lines.append("")
+        lines.append(f"BOTTOM LINE: {e['bottom_line']}")
+    caveat = policy_caveat_lines(e)     # held for review; absent on the baseline
+    if caveat:
+        lines.append("")
+        lines += caveat
+    lines.append("")
+    lines.append(e["analysis"])
+    if e.get("state_variations"):
+        named = [v for v in e["state_variations"]
+                 if v["state"].lower() != "general"]
+        shown = named if max_states is None else named[:max_states]
+        lines.append("")
+        if max_states is not None and len(named) > len(shown):
+            from collections import Counter
+            counts = Counter(v.get("is_conviction", "depends") for v in named)
+            lines.append(f"State-by-state coverage: {len(named)} jurisdictions "
+                         f"({', '.join(f'{k}: {n}' for k, n in counts.most_common())}) "
+                         f"-- full per-state detail in the per-state entries.")
+        else:
+            lines.append("State variations:")
+        for v in shown:
+            flag = (f" (conviction: {_vlabel(v['is_conviction'])})"
+                    if v.get("is_conviction") else "")
+            lines += _row_lines(v, flag)
+        for v in e["state_variations"]:
+            if v["state"].lower() == "general":
+                lines.append(f"- General: {v['note']}")
+        lines.append("")
+        lines.append(SURVEY_NOTE)
+    # The Saini checklist (roadmap T2.8), only for an entry that carries the
+    # curated 'pardon_test' (held for review; the approved baseline has none).
+    from canlex.screening import saini_checklist
+    checklist = saini_checklist(e)
+    if checklist:
+        lines.append("")
+        lines += checklist
+    if e.get("authorities"):
+        # each followed by the authority registry's brief pointer (roadmap
+        # T3.2; the bare bullets when the registry is not rendered)
+        from canlex.authorities import authority_lines
+        lines.append("")
+        lines.append("Authorities:")
+        lines += authority_lines(e["authorities"], brief=True)
+    if e.get("guidance"):
+        lines.append("")
+        lines.append(GUIDANCE_HEADING)
+        for g in e["guidance"]:
+            lines.append(f"- {g['ref']}: {g['note']}")
+    if e.get("interpretation"):
+        lines.append("")
+        lines.append("INTERPRETATION (no direct authority -- this is CanLex's "
+                     "reasoned view from the governing principles; treat it as "
+                     "a starting point, not an answer): "
+                     + e["interpretation"])
+    return "\n".join(lines)
+
+
+def build(allow_shrink=False):
+    data = json.loads(CURATED.read_text(encoding="utf-8"))
+    chunks = []
+    for e in data.get("methodology", []):
+        chunks.append({
+            "id": f"commentary-method-{e['id']}",
+            "doc_type": "commentary",
+            "act_code": ACT_CODE,
+            "act_short": ACT_SHORT,
+            "act_name": ACT_NAME,
+            "section": e["id"],
+            "marginal_note": e["title"],
+            "part": "Methodology",
+            "division": "",
+            "heading": e["title"],
+            "text": BANNER + "\n\n" + e["text"],
+            "history": "",
+            "last_amended": "",
+            "current_to": data.get("reviewed", ""),
+            "citation": f"{ACT_SHORT} — {e['title']}",
+            "source_url": "",
+        })
+    for e in data.get("dispositions", []):
+        chunks.append({
+            "id": f"commentary-disp-{e['id']}",
+            "doc_type": "commentary",
+            "act_code": ACT_CODE,
+            "act_short": ACT_SHORT,
+            "act_name": ACT_NAME,
+            "section": e["id"],
+            "marginal_note": e["names"][0],
+            "part": "US dispositions",
+            "division": "",
+            "heading": (f"Is a US {e['names'][0]} a conviction for IRPA "
+                        f"s. 36? ({_vlabel(e['is_conviction'])})"),
+            "text": _entry_text(e, max_states=6),
+            "history": "",
+            "last_amended": "",
+            "current_to": data.get("reviewed", ""),
+            "citation": f"{ACT_SHORT} — {e['names'][0]}",
+            "source_url": "",
+        })
+
+    # One chunk per jurisdiction: every disposition row for that state, so a
+    # query naming a state ("Georgia first offender act", "Missouri SIS")
+    # retrieves that state's page directly.
+    by_state = {}
+    for e in data.get("dispositions", []):
+        for v in e.get("state_variations", []):
+            st = v["state"]
+            if st.lower() == "general":
+                continue
+            flag = _vlabel(v.get("is_conviction") or e["is_conviction"])
+            row = dict(v, state=e["names"][0])
+            by_state.setdefault(st, []).extend(
+                _row_lines(row, f" (conviction: {flag})"))
+    for st in sorted(by_state):
+        body = (BANNER + "\n\n"
+                + f"US dispositions — {st}: whether each disposition type is a "
+                  f"conviction for IRPA s. 36, under {st} law.\n\n"
+                + "\n".join(by_state[st])
+                + "\n\n" + SURVEY_NOTE
+                + "\n\nThe act branch (IRPA s. 36(1)(c)/(2)(c)) can apply even "
+                  "where a disposition is not a conviction. See the "
+                  "per-disposition entries for the governing analysis and "
+                  "authorities.")
+        slug = st.lower().replace(" ", "-")
+        chunks.append({
+            "id": f"commentary-state-{slug}",
+            "doc_type": "commentary",
+            "act_code": ACT_CODE,
+            "act_short": ACT_SHORT,
+            "act_name": ACT_NAME,
+            "section": f"state-{slug}",
+            "marginal_note": f"US dispositions — {st}",
+            "part": "US dispositions by state",
+            "division": "",
+            "heading": (f"{st}: criminal dispositions vs the IRPA "
+                        f"'conviction' concept"),
+            "text": body,
+            "history": "",
+            "last_amended": "",
+            "current_to": data.get("reviewed", ""),
+            "citation": f"{ACT_SHORT} — {st}",
+            "source_url": "",
+        })
+    # --- equivalency pairings (step 2), same banner discipline
+    n_pairings = 0
+    if EQUIV.exists():
+        eq = json.loads(EQUIV.read_text(encoding="utf-8"))
+        for m in eq.get("methodology", []):
+            chunks.append({
+                "id": f"commentary-method-{m['id']}",
+                "doc_type": "commentary",
+                "act_code": ACT_CODE,
+                "act_short": ACT_SHORT,
+                "act_name": ACT_NAME,
+                "section": m["id"],
+                "marginal_note": m["title"],
+                "part": "Methodology",
+                "division": "",
+                "heading": m["title"],
+                "text": BANNER + "\n\n" + m["text"],
+                "history": "",
+                "last_amended": "",
+                "current_to": eq.get("reviewed", ""),
+                "citation": f"{ACT_SHORT} — {m['title']}",
+                "source_url": "",
+            })
+        for p in eq.get("pairings", []):
+            n_pairings += 1
+            lines = [BANNER, "",
+                     f"US offence: {'; '.join(p['us_terms'][:5])}",
+                     f"Canadian equivalent: {p['canadian_offence']}",
+                     f"Maximum penalty (verified against the corpus text): "
+                     f"{p['penalty']}",
+                     f"Inadmissibility branch: {p['branch']}",
+                     f"Authority status: {_STATUS_LABEL[p['status']]}", "",
+                     p["analysis"]]
+            if p.get("caveats"):
+                lines.append("")
+                lines.append("Caveats:")
+                lines += [f"- {c}" for c in p["caveats"]]
+            if p.get("authorities"):
+                from canlex.authorities import authority_lines
+                lines.append("")
+                lines.append("Authorities:")
+                lines += authority_lines(p["authorities"], brief=True)
+            chunks.append({
+                "id": f"commentary-equiv-{p['id']}",
+                "doc_type": "commentary",
+                "act_code": ACT_CODE,
+                "act_short": ACT_SHORT,
+                "act_name": ACT_NAME,
+                "section": f"equiv-{p['id']}",
+                "marginal_note": f"Equivalency: {p['us_terms'][0]}",
+                "part": "US offence equivalency",
+                "division": "",
+                "heading": (f"What does a US {p['us_terms'][0]} conviction "
+                            f"equate to in Canada?"),
+                "text": "\n".join(lines),
+                "history": "",
+                "last_amended": "",
+                "current_to": eq.get("reviewed", ""),
+                "citation": f"{ACT_SHORT} — equivalency: {p['us_terms'][0]}",
+                "source_url": "",
+            })
+
+    # A full re-render of the curated files, so a truncated or half-edited
+    # us_dispositions.json (or an equivalency file that moved) collapses the
+    # chunk count and the write would replace good analysis with the remnant.
+    # indent=1 is the stored file's format -- changing it rewrites every line.
+    if not write_corpus(OUT, chunks, indent=1, allow_shrink=allow_shrink):
+        return False
+    print(f"{len(chunks)} commentary chunks "
+          f"({len(data.get('dispositions', []))} dispositions, "
+          f"{len(by_state)} state pages, {n_pairings} equivalency pairings, "
+          f"{len(data.get('methodology', []))}+ methodology) -> {OUT}")
+    return True
+
+
+def main():
+    sys.exit(0 if build(allow_shrink="--allow-shrink" in sys.argv) else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/conduct.py

@@ -0,0 +1,401 @@
+"""Ingest public policy pages: employer policy instruments and a few
+operational-policy and treaty texts (one chunk per heading).
+
+Nearly every CBSA discipline letter and FPSLREB discipline decision in the
+corpus is framed as a breach of a numbered section of the CBSA Code of
+Conduct, and the Code itself operates under the Treasury Board's Values and
+Ethics Code for the Public Sector -- yet neither text was in the corpus, so
+a steward asking what the Code says about off-duty conduct or outside
+employment got tribunal paraphrases of older editions. The Treasury Board
+Directive on Terms and Conditions of Employment joins them: the FB agreement
+defers to it for pay administration. All are single public HTML pages; each
+is parsed into one chunk per heading.
+
+These are employer policy instruments: binding on employees as a condition
+of employment (the CBSA Code requires an annual attestation), grievable
+when applied, but neither legislation nor part of the collective agreement.
+server._format_section banners doc_type='policy' accordingly.
+
+Each DOCS entry is one instrument: `url` (or `urls`, for a document
+published as several pages), optional `doc_type` ('policy' by default;
+'memorandum' for CBSA operational policy, 'legislation' for the Safe Third
+Country Agreement), optional `lang` ('fr' twins carry a '-fr-' id infix and
+lang='fr' like every other French chunk), and `powershell` for canada.ca
+hosts that reject Python's TLS client.
+
+    python -m canlex.conduct [--force] [--allow-shrink]
+"""
+import re
+import sys
+
+from bs4 import BeautifulSoup
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "policy"
+OUT_FILE = PROCESSED_DIR / "policy.json"
+
+_TBS = "https://www.tbs-sct.canada.ca/pol/doc-{lang}.aspx?id={id}"
+_CBSA_DET = "https://www.cbsa-asfc.gc.ca/security-securite/detent/"
+_STCA = ("https://www.canada.ca/en/immigration-refugees-citizenship/corporate/"
+         "mandate/policies-operational-instructions-agreements/agreements/"
+         "safe-third-country-agreement/")
+
+
+def _tbs(code, short, name, id, lang="en"):
+    """A Treasury Board policy-suite instrument, English or French."""
+    return {"code": code, "short": short, "name": name,
+            "url": _TBS.format(lang="eng" if lang == "en" else "fra", id=id),
+            "raw": f"tbs-{id}-{lang}.html", "lang": lang}
+
+
+DOCS = [
+    {"code": "CBSA-CODE",
+     "short": "CBSA Code of Conduct",
+     "name": "Canada Border Services Agency Code of Conduct",
+     "url": "https://www.cbsa-asfc.gc.ca/agency-agence/reports-rapports/"
+            "acc-resp/code-eng.html",
+     "raw": "cbsa-code-eng.html"},
+    {"code": "VE-CODE",
+     "short": "Values and Ethics Code",
+     "name": "Values and Ethics Code for the Public Sector (Treasury Board)",
+     "url": "https://www.tbs-sct.canada.ca/pol/doc-eng.aspx?id=25049",
+     "raw": "values-ethics-code-eng.html"},
+    # The FB agreement defers to this Directive for pay on promotion and
+    # acting, increments, hours-of-work defaults and overpayment recovery.
+    {"code": "TB-TCE",
+     "short": "Terms and Conditions Directive",
+     "name": "Directive on Terms and Conditions of Employment (Treasury Board)",
+     "url": "https://www.tbs-sct.canada.ca/pol/doc-eng.aspx?id=15772",
+     "raw": "tb-terms-conditions-eng.html"},
+    # --- the people-management instruments the FPSLREB decisions apply ---
+    _tbs("TB-DISC", "Guidelines for Discipline",
+         "Guidelines for Discipline (Treasury Board)", 22370),
+    _tbs("TB-DTA", "Duty to Accommodate Directive",
+         "Directive on the Duty to Accommodate (Treasury Board)", 32634),
+    _tbs("TB-PERF", "Performance Management Directive",
+         "Directive on Performance Management (Treasury Board)", 27146),
+    _tbs("TB-WPHV", "Harassment and Violence Directive",
+         "Directive on the Prevention and Resolution of Workplace Harassment "
+         "and Violence (Treasury Board)", 32671),
+    _tbs("TB-LEAVE", "Leave and Special Working Arrangements Directive",
+         "Directive on Leave and Special Working Arrangements (Treasury "
+         "Board)", 15774),
+    _tbs("TB-PGS", "Policy on Government Security",
+         "Policy on Government Security (Treasury Board)", 16578),
+    _tbs("TB-SEC", "Security Screening Directive",
+         "Directive on Security Screening (Treasury Board, in effect "
+         "2025-01-06; replaces the Standard on Security Screening)", 32805),
+    # --- French twins: the same instruments as their francophone readers
+    # get them (lang='fr'; an English query never sees them) ---
+    {"code": "CBSA-CODE", "lang": "fr",
+     "short": "CBSA Code of Conduct",
+     "name": "Code de conduite de l'Agence des services frontaliers du Canada",
+     "url": "https://www.cbsa-asfc.gc.ca/agency-agence/reports-rapports/"
+            "acc-resp/code-fra.html",
+     "raw": "cbsa-code-fra.html"},
+    _tbs("VE-CODE", "Values and Ethics Code",
+         "Code de valeurs et d'éthique du secteur public", 25049, "fr"),
+    _tbs("TB-TCE", "Terms and Conditions Directive",
+         "Directive sur les conditions d'emploi", 15772, "fr"),
+    _tbs("TB-DISC", "Guidelines for Discipline",
+         "Lignes directrices concernant la discipline", 22370, "fr"),
+    _tbs("TB-DTA", "Duty to Accommodate Directive",
+         "Directive sur l'obligation de prendre des mesures d'adaptation",
+         32634, "fr"),
+    _tbs("TB-PERF", "Performance Management Directive",
+         "Directive sur la gestion du rendement", 27146, "fr"),
+    _tbs("TB-WPHV", "Harassment and Violence Directive",
+         "Directive sur la prévention et la résolution du harcèlement et de "
+         "la violence dans le lieu de travail", 32671, "fr"),
+    _tbs("TB-LEAVE", "Leave and Special Working Arrangements Directive",
+         "Directive sur les congés et les modalités de travail spéciales",
+         15774, "fr"),
+    _tbs("TB-PGS", "Policy on Government Security",
+         "Politique sur la sécurité du gouvernement", 16578, "fr"),
+    _tbs("TB-SEC", "Security Screening Directive",
+         "Directive sur le filtrage de sécurité", 32805, "fr"),
+    # --- CBSA / Public Safety detention policy: operational guidance, so
+    # doc_type 'memorandum' (persuasive, not binding) like the ENF chapters ---
+    {"code": "CBSA-NIDF", "doc_type": "memorandum",
+     "short": "CBSA Detention Framework",
+     "name": "CBSA National Immigration Detention Framework",
+     "url": _CBSA_DET + "nidf-cndi-eng.html", "raw": "cbsa-nidf-eng.html"},
+    {"code": "CBSA-NDDHM", "doc_type": "memorandum",
+     "short": "CBSA Minors Detention Directive",
+     "name": "CBSA National Directive for the Detention or Housing of Minors",
+     "url": _CBSA_DET + "nddhm-dndhm-eng.html", "raw": "cbsa-nddhm-eng.html"},
+    {"code": "CBSA-NIDS", "doc_type": "memorandum",
+     "short": "CBSA Detention Standards",
+     "name": "CBSA National Immigration Detention Standards (chapters 1-6)",
+     "urls": [_CBSA_DET + f"standards-normes/ch{n}-eng.html" for n in range(1, 7)],
+     "raw": "cbsa-nids-eng"},
+    # --- the IRCC applicant guide the screening chain points people to:
+    # eligibility, the five-year calculation with worked examples, fees,
+    # where to apply. Guidance (doc_type 'memorandum'), like ENF 14 / OP 19.
+    {"code": "IMM-5312", "doc_type": "memorandum", "powershell": True,
+     "short": "Guide 5312",
+     "name": "IRCC Guide 5312 -- Rehabilitation for Persons Who Are "
+             "Inadmissible to Canada Because of Past Criminal Activity "
+             "(application guide for form IMM 1444)",
+     "url": "https://www.canada.ca/en/immigration-refugees-citizenship/"
+            "services/application/application-forms-guides/"
+            "guide-5312-rehabilitation-persons-inadmissible-canada-past-"
+            "criminal-activity.html",
+     "raw": "imm5312-guide-eng.html"},
+    # --- IRCC's applicant-facing pages on overcoming criminal
+    # inadmissibility (added 2026-09-12, roadmap T3.8). The rehabilitation
+    # memo names form IMM 1444: its landing page carries the form's
+    # supplementary instructions. The form PDF itself is an XFA form whose
+    # text layer is only a 'Please wait...' viewer shell, so it is not
+    # ingested. Applicant-facing summaries, not officer instructions:
+    # doc_type 'memorandum' like Guide 5312; the law is IRPR ss. 17-18.
+    {"code": "IMM-1444", "doc_type": "memorandum", "powershell": True,
+     "short": "IMM 1444 form page",
+     "name": "IRCC Application for Criminal Rehabilitation (IMM 1444) -- "
+             "form page with the form's supplementary instructions "
+             "(applicant-facing)",
+     "url": "https://www.canada.ca/en/immigration-refugees-citizenship/"
+            "services/application/application-forms-guides/imm1444.html",
+     "raw": "imm1444-page-eng.html"},
+    {"code": "IRCC-PUB-OCC", "doc_type": "memorandum", "powershell": True,
+     "short": "IRCC: Overcome criminal convictions",
+     "name": "IRCC public page: Overcome criminal convictions "
+             "(applicant-facing)",
+     "url": "https://www.canada.ca/en/immigration-refugees-citizenship/"
+            "services/admissibility-enforcement/"
+            "overcome-criminal-convictions.html",
+     "raw": "ircc-overcome-criminal-convictions-eng.html"},
+    {"code": "IRCC-PUB-DEEMED", "doc_type": "memorandum", "powershell": True,
+     "short": "IRCC: Deemed rehabilitation",
+     "name": "IRCC public page: Deemed rehabilitation (applicant-facing)",
+     "url": "https://www.canada.ca/en/immigration-refugees-citizenship/"
+            "services/admissibility-enforcement/"
+            "overcome-criminal-convictions/deemed-rehabilitation.html",
+     "raw": "ircc-deemed-rehabilitation-eng.html"},
+    {"code": "PS-MD-MINORS", "doc_type": "memorandum",
+     "short": "Ministerial Direction on Minors in Detention",
+     "name": "Ministerial Direction to the CBSA: Minors in Canada's "
+             "Immigration Detention System (Public Safety, November 2017)",
+     "url": "https://www.publicsafety.gc.ca/cnt/trnsprnc/ns-trnsprnc/"
+            "mnstrl-drctn-cbsa-en.aspx",
+     "raw": "ps-md-minors-eng.html"},
+    # --- the Canada-US Safe Third Country Agreement: a treaty implemented by
+    # IRPR ss. 159.1-159.7, so it renders as law, named as a treaty ---
+    {"code": "STCA", "doc_type": "legislation", "powershell": True,
+     "short": "STCA",
+     "name": "Safe Third Country Agreement (Canada-United States treaty, in "
+             "force 2004-12-29; implemented by IRPR ss. 159.1-159.7)",
+     "url": _STCA + "final-text.html", "raw": "stca-final-text-eng.html",
+     "current_to": "2004-12-29"},
+    {"code": "STCA-PROTOCOL", "doc_type": "legislation", "powershell": True,
+     "short": "STCA Additional Protocol",
+     "name": "Additional Protocol to the Safe Third Country Agreement "
+             "(in force 2023-03-25: claims between ports of entry)",
+     "url": _STCA + "additional-protocol.html",
+     "raw": "stca-additional-protocol-eng.html", "current_to": "2023-03-25"},
+]
+
+# canada.ca stamps the date two ways: a visible <time property="dateModified">
+# on the WET template, and a <meta name="dcterms.modified" content="..."> on
+# the application-guide pages (which have no visible stamp at all).
+_MODIFIED = re.compile(
+    r'dateModified"[^>]*>(\d{4}-\d{2}-\d{2})|'
+    r'dcterms\.modified"[^>]*content="(\d{4}-\d{2}-\d{2})"')
+_EFFECTIVE = re.compile(
+    r"(?:effective|in effect|takes? effect|comes? into (?:effect|force)|"
+    r"came into (?:effect|force))\D{0,40}"
+    r"((?:January|February|March|April|May|June|July|August|September|"
+    r"October|November|December)\s+\d{1,2},\s+\d{4})", re.I)
+# Site chrome and navigation that sit inside <main> on canada.ca pages.
+_SKIP = {"on this page", "page details", "contacts", "footnotes",
+         "language selection", "search", "menu", "you are here:",
+         "about this site", "government of canada", "date modified:",
+         "report a problem or mistake on this page", "share this page",
+         "government of canada activities and initiatives",
+         "about government", "topics menu", "search and menus",
+         "table of contents",
+         # canada.ca form pages: how to run Acrobat, not what the form asks
+         "how to download, save and open this form",
+         "open this form in acrobat reader",
+         # TBS policy-suite chrome
+         "supporting tools", "hierarchy", "archives", "more information",
+         # the same chrome in French
+         "sur cette page", "détails de la page", "table des matières",
+         "notes de bas de page", "sélection de la langue", "recherche",
+         "vous êtes ici :", "à propos de ce site", "gouvernement du canada",
+         "date de modification :", "signaler un problème ou une erreur sur "
+         "cette page", "partagez cette page", "activités et initiatives du "
+         "gouvernement du canada", "à propos du gouvernement",
+         "menu des sujets", "recherche et menus", "outils de soutien",
+         "hiérarchie", "renseignements supplémentaires",
+         "personnes-ressources", "coordonnées"}
+_HEADINGS = ("h2", "h3", "h4")
+_CONTENT = {"p", "blockquote", "table", "dl", "pre", "figure"}
+_PIECE = 1800
+
+
+def _own_text(li):
+    """A list item's own text, without its nested lists (which are rendered
+    as their own bullets): the V&E Code nests '1.1 ...' items under a
+    '<strong>Respect For Democracy</strong>' item."""
+    return _norm(" ".join(
+        child.get_text(" ", strip=True) if getattr(child, "name", None) else str(child)
+        for child in li.children
+        if getattr(child, "name", None) not in ("ul", "ol")))
+
+
+def _block_text(heading):
+    """Readable text from a heading up to the next heading of any level."""
+    lines = []
+    for sib in heading.find_next_siblings():
+        if sib.name in _HEADINGS or sib.name == "h1":
+            break
+        if sib.name in ("ul", "ol"):
+            for li in sib.find_all("li"):
+                item = _own_text(li)
+                if item:
+                    lines.append(f"- {item}")
+        elif sib.name == "dl":
+            # Definition lists (the Directive's Part 1 is one 9 KB <dl>):
+            # one 'term: definition' line each, so the block can split.
+            term = ""
+            for el in sib.find_all(["dt", "dd"]):
+                text = _norm(el.get_text(" ", strip=True))
+                if el.name == "dt":
+                    term = text
+                elif text:
+                    lines.append(f"{term}: {text}" if term else text)
+        elif sib.name in _CONTENT:
+            text = _norm(sib.get_text(" ", strip=True))
+            if text:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def parse(html, doc, url=None, start=0):
+    """Chunks for one page of a document: one per h2/h3/h4 heading with body
+    text, part = the enclosing h2 (chapter). `start` continues the section
+    numbering across a multi-page document. Pure, for tests."""
+    url = url or doc.get("url") or doc["urls"][0]
+    lang = doc.get("lang", "en")
+    prefix = f"policy-fr-{doc['code'].lower()}" if lang == "fr" else \
+        f"policy-{doc['code'].lower()}"
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main") or soup
+    # canada.ca wraps sections in div/section; the TBS policy suite wraps
+    # each chapter in <details><summary><h2>..</h2></summary><div>..</div>
+    # and appendix headings in <section><header><h3>..</h3></header>.
+    # Flatten all of it so each heading and its content become siblings.
+    for tag in main.find_all(["section", "div", "article", "details",
+                              "summary", "header"]):
+        tag.unwrap()
+    mod = _MODIFIED.search(html)
+    # A page without a dateModified stamp (the STCA texts) takes the date
+    # the DOCS entry pins -- for a treaty, its in-force date.
+    modified = ((mod.group(1) or mod.group(2)) if mod
+                else doc.get("current_to", ""))
+    eff = _EFFECTIVE.search(main.get_text(" ", strip=True))
+    effective = f"Effective {eff.group(1)}" if eff else ""
+    chunks, chapter, n = [], "", start
+    for h in main.find_all(list(_HEADINGS)):
+        title = re.sub(r"\s*\[\d+\]", "", _norm(h.get_text(" ", strip=True)))
+        if not title or title.lower() in _SKIP:
+            continue
+        if h.name == "h2":
+            chapter = title
+        body = _block_text(h)
+        if len(body) < 80:
+            continue
+        n += 1
+        pieces = split_lines(body, _PIECE) if len(body) > 2400 else [body]
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            chunk = {
+                "id": f"{prefix}-{n}{suffix}",
+                "doc_type": doc.get("doc_type", "policy"),
+                "act_code": doc["code"],
+                "act_short": doc["short"],
+                "act_name": doc["name"],
+                "section": title,
+                "marginal_note": title,
+                "part": chapter if h.name != "h2" else "",
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": effective,
+                "last_amended": modified,
+                "in_force": "",
+                "status": "in force",
+                "current_to": modified,
+                "citation": f"{doc['short']}{note}",
+                "source_url": url,
+            }
+            if lang == "fr":
+                chunk["lang"] = "fr"
+            chunks.append(chunk)
+    return chunks
+
+
+def _fetch_doc(doc, force=False):
+    """[(url, html)] for every page of the document."""
+    urls = doc.get("urls") or [doc["url"]]
+    pages = []
+    for i, url in enumerate(urls, start=1):
+        raw = doc["raw"] if len(urls) == 1 else f"{doc['raw']}-{i}.html"
+        html = fetch_cached(url, RAW / raw, pause=1.0, force=force,
+                            powershell=doc.get("powershell", False))
+        pages.append((url, html.decode("utf-8", "replace")))
+    return pages
+
+
+def build(allow_shrink=False, force=False):
+    chunks, failures = [], []
+    for doc in DOCS:
+        key = (doc["code"], doc.get("lang", "en"))
+        try:
+            pages = _fetch_doc(doc, force=force)
+        except Exception as exc:
+            print(f"  !! {doc['short']} ({key[1]}): fetch failed "
+                  f"({type(exc).__name__}: {exc})")
+            failures.append(key)
+            continue
+        got = []
+        for url, html in pages:
+            got += parse(html, doc, url=url, start=len(got))
+        if not got:
+            # Fetched, parsed to nothing: the page template moved. Keep the
+            # stored copy rather than dropping the instrument.
+            print(f"  !! {doc['short']} ({key[1]}): no content parsed; "
+                  f"keeping stored copy")
+            failures.append(key)
+            continue
+        chunks += got
+        print(f"  {doc['short']} ({key[1]}): {len(got)} chunks, modified "
+              f"{got[0]['current_to'] or '?'}")
+    chunks, preserved = preserve_dropped(
+        chunks, stored_chunks(OUT_FILE),
+        key=lambda c: (c["act_code"], c.get("lang", "en")),
+        only=set(failures))
+    if preserved:
+        print(f"  kept the last-good copy of: "
+              f"{', '.join(f'{c} ({l})' for c, l in preserved)}")
+    uniquify_ids(chunks)
+    if not write_corpus(OUT_FILE, chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"conduct: {len(chunks)} chunks -> {OUT_FILE.name}")
+    return chunks
+
+
+def main():
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv) is not False
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/config.py

@@ -0,0 +1,518 @@
+"""Paths and legislation sources for CanLex."""
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = ROOT / "data"
+RAW_DIR = DATA_DIR / "raw"
+PROCESSED_DIR = DATA_DIR / "processed"
+
+# Justice Laws Website XML sources. Each key is the Justice Laws consolidated
+# identifier, used as the XML filename and the local raw/processed filename.
+# To add an instrument, add an entry with a verified xml_url.
+#
+# The Constitution Acts, 1867 to 1982 are intentionally absent: Justice Laws
+# publishes them only as HTML/PDF, with no XML version available.
+SOURCES = {
+    # --- Immigration ---
+    "I-2.5": {
+        "short": "IRPA", "name": "Immigration and Refugee Protection Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/I-2.5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/i-2.5/"},
+    "SOR-2002-227": {
+        "short": "IRPR", "name": "Immigration and Refugee Protection Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2002-227.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2002-227/"},
+    "C-29": {
+        "short": "Citizenship Act", "name": "Citizenship Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-29.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-29/"},
+    # IRB tribunal procedural rules. (SOR-2002-230 was repealed because the
+    # Immigration Appeal Division Rules, 2022 -- SOR-2022-277, below --
+    # replaced it in January 2023.)
+    "SOR-2022-277": {
+        "short": "IAD Rules", "name": "Immigration Appeal Division Rules, 2022",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2022-277.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2022-277/"},
+    "SOR-2002-229": {
+        "short": "Immigration Division Rules", "name": "Immigration Division Rules",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2002-229.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2002-229/"},
+    "SOR-2012-256": {
+        "short": "RPD Rules", "name": "Refugee Protection Division Rules",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2012-256.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2012-256/"},
+    "SOR-2012-257": {
+        "short": "RAD Rules", "name": "Refugee Appeal Division Rules",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2012-257.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2012-257/"},
+
+    # --- Customs & trade ---
+    "C-52.6": {
+        "short": "Customs Act", "name": "Customs Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-52.6.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-52.6/"},
+    # skip_schedules: the Customs Tariff's schedule is the entire tariff
+    # nomenclature -- tariff_schedule.py serves chapters 98/99 from the CBSA
+    # edition instead, so the XML schedule would only duplicate it at scale.
+    "C-54.011": {
+        "short": "Customs Tariff", "name": "Customs Tariff", "kind": "act",
+        "skip_schedules": True,
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-54.011.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-54.011/"},
+    # Traveller/trade instruments under the Customs Act and EIPA. The two
+    # control lists are almost entirely Schedules, so they depend on the
+    # schedule parser; Presentation of Persons is the legal basis for NEXUS
+    # and every other alternative-presentation program.
+    "E-14.1": {
+        "short": "Excise Act, 2001", "name": "Excise Act, 2001", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/E-14.1.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/e-14.1/"},
+    # The Income Tax Act: added 2026-09-10 so the equivalency table's
+    # tax-evasion pairing (US 26 USC 7201 / 7206 -> ITA s. 239) rests on
+    # corpus-verified text like every other maximum. The largest instrument
+    # in the corpus (~14 MB of XML per language); its vocabulary ('resident',
+    # 'employee', 'benefit', 'duty', 'trust') overlaps the border and labour
+    # families, so it is measured question by question at every ingest.
+    "I-3.3": {
+        "short": "ITA", "name": "Income Tax Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/I-3.3.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/i-3.3/"},
+    "CRC-604": {
+        "short": "Import Control List", "name": "Import Control List",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C.R.C.,_c._604.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/C.R.C.,_c._604/"},
+    "SOR-89-202": {
+        "short": "Export Control List", "name": "Export Control List",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-89-202.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-89-202/"},
+    "SOR-2003-323": {
+        "short": "Presentation of Persons Regs",
+        "name": "Presentation of Persons (2003) Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2003-323.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2003-323/"},
+    "SOR-86-1072": {
+        "short": "Duty Free Shop Regs", "name": "Duty Free Shop Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-1072.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-1072/"},
+    "SOR-86-1065": {
+        "short": "Sufferance Warehouses Regs",
+        "name": "Customs Sufferance Warehouses Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-1065.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-1065/"},
+    "S-8.35": {
+        "short": "Luxury Tax Act", "name": "Select Luxury Items Tax Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-8.35.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/s-8.35/"},
+    "C-1.4": {
+        "short": "CBSA Act", "name": "Canada Border Services Agency Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-1.4.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-1.4/"},
+    "S-15": {
+        "short": "SIMA", "name": "Special Import Measures Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-15.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/s-15/"},
+    "E-19": {
+        "short": "EIPA", "name": "Export and Import Permits Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/E-19.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/e-19/"},
+    "P-19.32": {
+        "short": "Preclearance Act", "name": "Preclearance Act, 2016", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-19.32.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-19.32/"},
+    # Customs Act operational regulations -- the procedural detail behind the
+    # bare Act (reporting, accounting, exemptions, and the designated-provisions
+    # regulation that is the legal hook for every AMPS monetary penalty).
+    "SOR-86-873": {
+        "short": "Reporting of Imported Goods Regs",
+        "name": "Reporting of Imported Goods Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-873.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-873/"},
+    "SOR-86-1062": {
+        "short": "Accounting for Imported Goods Regs",
+        "name": "Accounting for Imported Goods and Payment of Duties Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-1062.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-1062/"},
+    "SOR-2002-336": {
+        "short": "Designated Provisions (Customs) Regs",
+        "name": "Designated Provisions (Customs) Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2002-336.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2002-336/"},
+    "SOR-96-46": {
+        "short": "Customs Bonded Warehouses Regs",
+        "name": "Customs Bonded Warehouses Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-96-46.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-96-46/"},
+    "SOR-98-58": {
+        "short": "Returning Persons Exemption Regs",
+        "name": "Returning Persons Exemption Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-98-58.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-98-58/"},
+
+    # --- Criminal & financial crime ---
+    "C-46": {
+        "short": "Criminal Code", "name": "Criminal Code", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-46.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-46/"},
+    "F-11.6": {
+        "short": "Firearms Act", "name": "Firearms Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/F-11.6.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/f-11.6/"},
+    # Made under Criminal Code s. 117.15: the model-by-model prohibited/
+    # restricted/non-restricted classifications (incl. the May 2020
+    # SOR/2020-96 prohibitions). Nearly all substance is in its Schedules --
+    # this entry is why the schedule parser exists.
+    "SOR-98-462": {
+        "short": "Firearms Classification Regs",
+        "name": ("Regulations Prescribing Certain Firearms and Other Weapons, "
+                 "Components and Parts of Weapons, Accessories, Cartridge "
+                 "Magazines, Ammunition and Projectiles as Prohibited, "
+                 "Restricted or Non-Restricted"), "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-98-462.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-98-462/"},
+    "P-24.501": {
+        "short": "PCMLTFA",
+        "name": "Proceeds of Crime (Money Laundering) and Terrorist Financing Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-24.501.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-24.501/"},
+    "SOR-2002-412": {
+        "short": "Cross-border Currency Reporting Regs",
+        "name": "Cross-border Currency and Monetary Instruments Reporting Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2002-412.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2002-412/"},
+    "S-6.9": {
+        "short": "SCIDA", "name": "Security of Canada Information Disclosure Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-6.9.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/s-6.9/"},
+
+    # --- Controlled substances & cannabis ---
+    "C-38.8": {
+        "short": "CDSA", "name": "Controlled Drugs and Substances Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-38.8.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-38.8/"},
+    "SOR-97-234": {
+        "short": "CDSA Police Enforcement Regs",
+        "name": "Controlled Drugs and Substances Act (Police Enforcement) Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-97-234.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-97-234/"},
+    "SOR-2000-217": {
+        "short": "Benzodiazepines Regs",
+        "name": "Benzodiazepines and Other Targeted Substances Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2000-217.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2000-217/"},
+    "C.R.C.,_c._1041": {
+        "short": "Narcotic Control Regs", "name": "Narcotic Control Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C.R.C.%2C_c._1041.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/C.R.C.%2C_c._1041/"},
+    "SOR-2012-230": {
+        "short": "New Classes of Practitioners Regs",
+        "name": "New Classes of Practitioners Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2012-230.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2012-230/"},
+    "SOR-2002-359": {
+        "short": "Precursor Control Regs", "name": "Precursor Control Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2002-359.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2002-359/"},
+    "SOR-97-229": {
+        "short": "Precursor & Substance Exemption Regs",
+        "name": "Regulations Exempting Certain Precursors and Controlled Substances "
+                "from the Application of the Controlled Drugs and Substances Act",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-97-229.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-97-229/"},
+    "C-24.5": {
+        "short": "Cannabis Act", "name": "Cannabis Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-24.5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-24.5/"},
+    "SOR-2018-144": {
+        "short": "Cannabis Regs", "name": "Cannabis Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2018-144.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2018-144/"},
+
+    # --- Food, health, agriculture ---
+    "F-27": {
+        "short": "Food and Drugs Act", "name": "Food and Drugs Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/F-27.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/f-27/"},
+    "C.R.C.,_c._870": {
+        "short": "Food and Drug Regs", "name": "Food and Drug Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C.R.C.%2C_c._870.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/C.R.C.%2C_c._870/"},
+    "H-3.3": {
+        "short": "Health of Animals Act", "name": "Health of Animals Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/H-3.3.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/h-3.3/"},
+    "C.R.C.,_c._296": {
+        "short": "Health of Animals Regs", "name": "Health of Animals Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C.R.C.%2C_c._296.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/C.R.C.%2C_c._296/"},
+    "P-14.8": {
+        "short": "Plant Protection Act", "name": "Plant Protection Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-14.8.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-14.8/"},
+    "SOR-95-212": {
+        "short": "Plant Protection Regs", "name": "Plant Protection Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-95-212.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-95-212/"},
+    "SOR-2018-108": {
+        "short": "Safe Food for Canadians Regs",
+        "name": "Safe Food for Canadians Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2018-108.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2018-108/"},
+    "A-8.8": {
+        "short": "AAAMPA",
+        "name": "Agriculture and Agri-Food Administrative Monetary Penalties Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/A-8.8.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/a-8.8/"},
+    "Q-1.1": {
+        "short": "Quarantine Act", "name": "Quarantine Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/Q-1.1.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/q-1.1/"},
+    # CITES enforcement at the border -- the basis for seizures of protected
+    # wildlife, ivory and exotic-species goods.
+    "W-8.5": {
+        "short": "WAPPRIITA",
+        "name": "Wild Animal and Plant Protection and Regulation of "
+                "International and Interprovincial Trade Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/W-8.5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/w-8.5/"},
+    "SOR-96-263": {
+        "short": "WAPTR",
+        "name": "Wild Animal and Plant Trade Regulations", "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-96-263.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-96-263/"},
+
+    # --- Labour & public service ---
+    # CHRA grounds the accommodation/discrimination case law already in the
+    # corpus (Renaud, Hydro-Québec, Elk Valley); GECA governs injury-on-duty
+    # compensation interlocking with the FB agreement's IOD leave.
+    "H-6": {
+        "short": "CHRA", "name": "Canadian Human Rights Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/H-6.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/h-6/"},
+    "G-5": {
+        "short": "GECA", "name": "Government Employees Compensation Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/G-5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/g-5/"},
+    "L-2": {
+        "short": "Canada Labour Code", "name": "Canada Labour Code", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/L-2.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/l-2/"},
+    # The whole federal harassment/violence process (notice of occurrence,
+    # designated recipient, resolution, investigation, timelines) lives in
+    # these Regulations since 2021, not in CLC Part II itself; Marentette
+    # 2024 FC 676 (in the corpus) interprets them.
+    "SOR-2020-130": {
+        "short": "WPHV Regs",
+        "name": "Work Place Harassment and Violence Prevention Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2020-130.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2020-130/"},
+    # Part II of the Code binds the federal public administration (s. 123(2));
+    # these two Regulations give its duties their content -- first aid,
+    # lighting, sanitation, hazardous substances, confined spaces (COHSR) --
+    # and constitute the policy/work-place committees a steward sits on.
+    "SOR-86-304": {
+        "short": "COHSR",
+        "name": "Canada Occupational Health and Safety Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-86-304.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-86-304/"},
+    "SOR-2015-164": {
+        "short": "OHS Committees Regs",
+        "name": "Policy Committees, Work Place Committees and Health and "
+                "Safety Representatives Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2015-164.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2015-164/"},
+    "P-33.3": {
+        "short": "FPSLRA", "name": "Federal Public Sector Labour Relations Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-33.3.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-33.3/"},
+    "P-33.35": {
+        "short": "FPSLREB Act",
+        "name": "Federal Public Sector Labour Relations and Employment Board Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-33.35.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-33.35/"},
+    "P-33.01": {
+        "short": "PSEA", "name": "Public Service Employment Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-33.01.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-33.01/"},
+    "P-36": {
+        "short": "PSSA", "name": "Public Service Superannuation Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-36.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-36/"},
+
+    # --- Privacy ---
+    # --- Public-service labour relations, discipline and disclosure ---
+    # The FPSLRA hands grievance time limits and adjudication procedure to
+    # its Regulations; FAA s. 12 is the legal authority for discipline in a
+    # department (and s. 155 the set-off behind overpayment recovery); the
+    # PSDPA underlies the CBSA Code's disclosure chapter and the V&E Code.
+    "SOR-2005-79": {
+        "short": "FPSLR Regulations",
+        "name": "Federal Public Sector Labour Relations Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2005-79.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2005-79/"},
+    "SOR-2005-334": {
+        "short": "PSER",
+        "name": "Public Service Employment Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2005-334.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2005-334/"},
+    "F-11": {
+        "short": "FAA",
+        "name": "Financial Administration Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/F-11.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/f-11/"},
+    "P-31.9": {
+        "short": "PSDPA",
+        "name": "Public Servants Disclosure Protection Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-31.9.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-31.9/"},
+    "O-3.01": {
+        "short": "OLA",
+        "name": "Official Languages Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/O-3.01.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/o-3.01/"},
+    "I-21": {
+        "short": "Interpretation Act",
+        "name": "Interpretation Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/I-21.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/i-21/"},
+    # --- Criminal records and youth: the other half of IRPA s. 36(3)(b),
+    # which lifts inadmissibility only while a record suspension 'has not
+    # ceased to have effect or been revoked' under the Criminal Records Act.
+    "C-47": {
+        "short": "Criminal Records Act",
+        "name": "Criminal Records Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-47.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-47/",
+        "skip_schedules": True},   # its offence-list schedules are keyword magnets
+    "Y-1.5": {
+        "short": "YCJA",
+        "name": "Youth Criminal Justice Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/Y-1.5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/y-1.5/"},
+    # --- Contraventions: IRPA s. 36(3)(e)(i) bars basing s. 36 inadmissibility
+    # on an offence 'designated as a contravention under the Contraventions
+    # Act'. The designations are the Regulations' schedules (skip_schedules
+    # stays off), so both are needed to check that caveat against corpus
+    # text. Added 2026-09-12 (roadmap T3.8).
+    "C-38.7": {
+        "short": "Contraventions Act",
+        "name": "Contraventions Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/C-38.7.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/c-38.7/"},
+    "SOR-96-313": {
+        "short": "Contraventions Regs",
+        "name": "Contraventions Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-96-313.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-96-313/"},
+    # --- Sanctions: IRPA s. 35(1)(c)-(e) inadmissibility is defined by
+    # reference to these Acts, and the CBSA enforces sanctioned goods.
+    "S-14.5": {
+        "short": "SEMA",
+        "name": "Special Economic Measures Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-14.5.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/s-14.5/"},
+    "U-2": {
+        "short": "United Nations Act",
+        "name": "United Nations Act",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/U-2.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/u-2/"},
+    "J-2.3": {
+        "short": "JVCFOA",
+        "name": "Justice for Victims of Corrupt Foreign Officials Act (Sergei Magnitsky Law)",
+        "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/J-2.3.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/j-2.3/"},
+    # --- Border regulations the D-memos and Preclearance Act point at ---
+    "SOR-2019-183": {
+        "short": "Preclearance in Canada Regs",
+        "name": "Preclearance in Canada Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2019-183.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2019-183/"},
+    "SOR-2003-219": {
+        "short": "Passenger Information Regs",
+        "name": "Passenger Information (Customs) Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2003-219.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2003-219/"},
+    "SOR-2005-23": {
+        "short": "Reporting of Exported Goods Regs",
+        "name": "Reporting of Exported Goods Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2005-23.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2005-23/"},
+    "SOR-98-209": {
+        "short": "Firearms Storage & Transport Regs",
+        "name": "Storage, Display, Transportation and Handling of Firearms by Individuals Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-98-209.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-98-209/"},
+    # --- Aviation security: the airport side of an officer's work. The
+    # Preclearance Act and its Regulations point at the aerodromes listed in
+    # the CASR 2012; the Aeronautics Act is their enabling Act; the Secure Air
+    # Travel Act and Regulations are the 'no-fly' list regime. The Identity
+    # Screening Regulations (SOR/2007-82) they replaced are wholly repealed
+    # (SOR/2015-182, s. 2) and are not added. Added 2026-10-12.
+    "A-2": {
+        "short": "Aeronautics Act", "name": "Aeronautics Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/A-2.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/a-2/"},
+    "SOR-2011-318": {
+        "short": "CASR 2012",
+        "name": "Canadian Aviation Security Regulations, 2012",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2011-318.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2011-318/"},
+    "S-6.7": {
+        "short": "SATA", "name": "Secure Air Travel Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/S-6.7.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/s-6.7/"},
+    "SOR-2015-181": {
+        "short": "Secure Air Travel Regs",
+        "name": "Secure Air Travel Regulations",
+        "kind": "regulation",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/SOR-2015-181.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/regulations/SOR-2015-181/"},
+    "P-21": {
+        "short": "Privacy Act", "name": "Privacy Act", "kind": "act",
+        "xml_url": "https://laws-lois.justice.gc.ca/eng/XML/P-21.xml",
+        "web_base": "https://laws-lois.justice.gc.ca/eng/acts/p-21/"},
+}

canlex/curated_checks.py

@@ -0,0 +1,578 @@
+"""The curated tables' statute phrases, checked against the corpus (roadmap T4.3).
+
+The curated tables pin the statute text they rely on with `verify` rows --
+(act, section, contains): the phrase must appear in that section as
+ingested. This module is the one implementation of that check. It is used by
+tests/test_curated_penalties.py and by `py -m canlex.refresh --check`
+(refresh.check_curated_sections), and is meant for tools/predeploy.py's gate
+(run_checks + gate). The refresh-drift helpers corpus_reader, verify_sections
+and refresh_drift moved here from tools/predeploy.py (wave 2); until predeploy
+and tools/compose_release.py import them from here, predeploy carries its own
+copy, and tests/test_refresh.py checks that the two agree.
+
+After a corpus refresh a verify row can show two signals:
+
+- broken: the phrase is no longer in its section, or the section is gone.
+  The table quotes text the statute no longer has.
+- amended: the section's last_amended (the corpus's own amendment date, read
+  with the section's `history` citation) is later than the date the review
+  saw. That seen date is the section's last_amended in the corpus of the
+  approved commit (data/curated/review_state.json), so `last_amended_seen`
+  advances when the maintainer approves a new baseline. It is reported per
+  verify row and never written into the curated JSON, which is held for
+  review.
+
+When an amended section's stated maximum changed in a subsection a verify row
+quotes, draft_history proposes a penalty_history row: the maximum before the
+amendment, until the day before the amendment. Under Tran, an offence
+committed before the amendment keeps the maximum then in force. No row is
+drafted when a row the pairing already carries covers the amendment. The
+drafts go to data/curated/drafts/ for review. Nothing here edits
+data/curated/*.json.
+"""
+import datetime
+import json
+import re
+import subprocess
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+CURATED = ROOT / "data" / "curated"
+DRAFTS = CURATED / "drafts"
+REVIEW_STATE = CURATED / "review_state.json"
+EQUIVALENCY = "us_equivalency.json"
+TRANSBORDER = "transborder.json"
+DRAFT_GLOB = "penalty-history-draft-*.json"
+
+PASS, WARN, FAIL = "PASS", "WARN", "FAIL"     # tools/predeploy.py's statuses
+
+
+# ---- corpus access (working tree or a git commit) -----------------------------
+
+def git_blob(commit, path, root=ROOT):
+    """The bytes of `path` at `commit`, or None when git cannot show it."""
+    try:
+        out = subprocess.run(["git", "-C", str(root), "show", f"{commit}:{path}"],
+                             capture_output=True, timeout=300)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    return out.stdout if out.returncode == 0 else None
+
+
+def lfs_content(blob, root=ROOT):
+    """The content an LFS pointer names, read from the LOCAL LFS store (never
+    fetched), or None when it is not there. A non-pointer blob is returned
+    as is (the same rule as tools/predeploy.py lfs_object)."""
+    if blob is None or not blob.startswith(b"version https://git-lfs"):
+        return blob
+    oid = next((l.split(b":", 1)[1].strip().decode() for l in blob.splitlines()
+                if l.startswith(b"oid sha256:")), None)
+    if not oid:
+        return None
+    try:
+        out = subprocess.run(["git", "-C", str(root), "rev-parse", "--git-common-dir"],
+                             capture_output=True, timeout=60)
+    except (OSError, subprocess.SubprocessError):
+        return None
+    common = Path(out.stdout.decode().strip())
+    if not common.is_absolute():
+        common = Path(root) / common
+    path = common / "lfs" / "objects" / oid[:2] / oid[2:4] / oid
+    return path.read_bytes() if path.exists() else None
+
+
+def _chunks(blob, root=ROOT):
+    blob = lfs_content(blob, root)
+    if blob is None:
+        return []
+    try:
+        return json.loads(blob.decode("utf-8"))
+    except ValueError:
+        return []
+
+
+def squash(text):
+    return re.sub(r"\s+", " ", text or "")
+
+
+class Corpus:
+    """data/processed read per act: the working tree (commit None), a git
+    commit (`git show`, LFS pointers read from the local store), or a dict of
+    chunks (tests)."""
+
+    def __init__(self, commit=None, root=ROOT, chunks=None):
+        self.commit, self.root = commit, Path(root)
+        self._cache = dict(chunks) if chunks is not None else {}
+        self._fixed = chunks is not None
+
+    def chunks(self, act):
+        if act not in self._cache:
+            if self._fixed:
+                return []
+            path = f"data/processed/{act}.json"
+            if self.commit is None:
+                p = self.root / path
+                blob = p.read_bytes() if p.exists() else None
+            else:
+                blob = git_blob(self.commit, path, self.root)
+            self._cache[act] = _chunks(blob, self.root)
+        return self._cache[act]
+
+    def section(self, act, section):
+        return [c for c in self.chunks(act) if str(c.get("section")) == str(section)]
+
+    def raw_text(self, act, section):
+        """The section's text with its line breaks (subsections start a line)."""
+        return "\n".join(c.get("text", "") for c in self.section(act, section))
+
+    def text(self, act, section):
+        """Whitespace squashed: the text a verify phrase is checked against."""
+        return squash(" ".join(c.get("text", "") for c in self.section(act, section)))
+
+    __call__ = text
+
+    def last_amended(self, act, section):
+        return max((c.get("last_amended") or "" for c in self.section(act, section)),
+                   default="")
+
+    def amended_by(self, act, section):
+        """The last citation in the section's history line ('2026, c. 19, s. 23')."""
+        hist = next((c.get("history") for c in self.section(act, section)
+                     if c.get("history")), "")
+        return hist.split(";")[-1].strip() if hist else ""
+
+
+def corpus_reader(commit=None, cwd=ROOT):
+    """section_text(act, section) over a commit's data/processed (None: the
+    working tree at cwd), whitespace squashed: the text a curated verify
+    phrase is checked against. Signature kept from tools/predeploy.py."""
+    return Corpus(commit, cwd).text
+
+
+# ---- verify rows ----------------------------------------------------------------
+
+def verify_rows(table, file=EQUIVALENCY):
+    """Every verify row in a curated table, flattened: a pairing's and its
+    limbs' (us_equivalency.json) or a row's (transborder.json). Each is
+    {file, owner, limb, act, section, contains}."""
+    out = []
+
+    def add(owner, limb, v):
+        out.append({"file": file, "owner": owner, "limb": limb, "act": v["act"],
+                    "section": str(v["section"]), "contains": v["contains"]})
+
+    for p in (table or {}).get("pairings") or []:
+        for v in p.get("verify") or []:
+            add(p.get("id"), None, v)
+        for l in p.get("limbs") or []:
+            if isinstance(l, dict):
+                for v in l.get("verify") or []:
+                    add(p.get("id"), l.get("key"), v)
+    for r in (table or {}).get("rows") or []:
+        for v in r.get("verify") or []:
+            add(r.get("id"), None, v)
+    return out
+
+
+def phrase_in(phrase, text):
+    """The phrase, whitespace squashed, occurs in the text (case-insensitive)."""
+    return " ".join((phrase or "").split()).lower() in (text or "").lower()
+
+
+def where(row):
+    return f"{row['owner']}" + (f" limb {row['limb']}" if row.get("limb") else "")
+
+
+def problems(rows, corpus):
+    """The broken rows as messages: '<owner>: <act> s. <section> lacks '<phrase>''
+    (or 'not in corpus'). [] when every phrase is in its section."""
+    out = []
+    for r in rows:
+        text = corpus.text(r["act"], r["section"])
+        if not text:
+            out.append(f"{where(r)}: {r['act']} s. {r['section']} not in corpus")
+        elif not phrase_in(r["contains"], text):
+            out.append(f"{where(r)}: {r['act']} s. {r['section']} lacks {r['contains']!r}")
+    return out
+
+
+# ---- refresh drift (moved from tools/predeploy.py, wave 2 verifier) -----------
+
+def verify_sections(pairing):
+    """{(act, section): the maxima the pairing's verify rows (and its limbs')
+    quote for that section} -- canlex.screening.maxima_stated on each
+    phrase."""
+    from .screening import maxima_stated
+    out = {}
+    rows = list(pairing.get("verify") or []) + [
+        v for l in pairing.get("limbs") or [] if isinstance(l, dict) for v in l.get("verify") or []]
+    for v in rows:
+        out.setdefault((v["act"], v["section"]), set()).update(maxima_stated(v.get("contains")))
+    return out
+
+
+def refresh_drift(equivalency, now_text, then_text):
+    """The curated table against a corpus refresh: (drift, changed).
+
+    drift: [(pairing id, act, section, sorted new maxima)] -- a maximum a
+    verify section states NOW that it did not state in the corpus the table
+    was approved against, and that the pairing does not quote (an amendment
+    the approved text cannot know: 2026, c. 19 raised s. 173(2)(a) to 10
+    years). A composed release shipping such a table ships stale law.
+    changed: [(pairing id, act, section)] -- every verify section whose text
+    changed at all (re-read before relying on the approved prose)."""
+    from .screening import maxima_stated
+    drift, changed = [], []
+    for p in (equivalency or {}).get("pairings") or []:
+        for (act, sec), quoted in sorted(verify_sections(p).items()):
+            now, then = now_text(act, sec), then_text(act, sec)
+            if not now or not then:
+                continue
+            if now != then:
+                changed.append((p["id"], act, sec))
+            new = maxima_stated(now) - maxima_stated(then) - quoted
+            if new:
+                drift.append((p["id"], act, sec, sorted(new)))
+    return drift, changed
+
+
+# ---- amendments and the maximum they changed ------------------------------------
+
+_SUBSECTION = re.compile(r"\n\((\d+(?:\.\d+)*)\)\s")
+
+
+def subsections(raw):
+    """{label: text} of a section's subsections as the corpus prints them (a
+    line opening '(1) ', '(2.1) '); the text before the first one, or a
+    section with none, is labelled ''. A section split over several chunks
+    is read as one text, so a chunk continuing a subsection stays in it."""
+    marks = list(_SUBSECTION.finditer(raw or ""))
+    if not marks:
+        return {"": raw or ""}
+    out = {}
+    head = raw[:marks[0].start()]
+    if head.strip():
+        out[""] = head
+    for i, m in enumerate(marks):
+        end = marks[i + 1].start() if i + 1 < len(marks) else len(raw)
+        out[m.group(1)] = out.get(m.group(1), "") + raw[m.start():end]
+    return out
+
+
+def maxima_seq(text):
+    """The maxima a text states, in order (canlex.screening.maxima_stated on
+    each stated term), so that a maximum moving between provisions shows."""
+    from . import screening as sc
+    return [x for m in sc._STATED_MAX.finditer((text or "").lower())
+            for x in sorted(sc.maxima_stated(m.group(0)))]
+
+
+def maximum_change(phrase, then_raw, now_raw):
+    """(changed, before, after, added, removed) for the subsections that hold
+    `phrase` then or now (every subsection when neither does).
+
+    changed: labels printed both then and now whose stated maxima differ as
+    sets, and only when the section's maxima as a whole differ (a
+    subsection split or renumbered around the same maxima is not a change).
+    before / after: the maxima those labels stated then / now.
+    added / removed: quoted labels only in the new / only in the old text,
+    a provision created or repealed, never read as a changed maximum."""
+    from collections import Counter
+    then_s, now_s = subsections(then_raw), subsections(now_raw)
+    labels = sorted({l for l, t in then_s.items() if phrase_in(phrase, squash(t))}
+                    | {l for l, t in now_s.items() if phrase_in(phrase, squash(t))})
+    if not labels:
+        labels = sorted(set(then_s) | set(now_s))
+    added = [l for l in labels if l not in then_s]
+    removed = [l for l in labels if l not in now_s]
+    # The unnumbered label is the whole section in a version without
+    # subsections, but only the number and head words in one with them: it is
+    # compared only when neither version numbers its subsections.
+    numbered = bool(set(then_s) - {""}) or bool(set(now_s) - {""})
+    changed = []
+    if Counter(maxima_seq(then_raw)) != Counter(maxima_seq(now_raw)):
+        changed = [l for l in labels if l in then_s and l in now_s
+                   and not (l == "" and numbered)
+                   and set(maxima_seq(then_s[l])) != set(maxima_seq(now_s[l]))]
+    before = sorted({x for l in changed for x in maxima_seq(then_s[l])})
+    after = sorted({x for l in changed for x in maxima_seq(now_s[l])})
+    return changed, before, after, added, removed
+
+
+def check_rows(rows, now, then=None):
+    """Each verify row with its report fields:
+
+    status          'broken' (phrase or section gone) > 'amended' > 'ok'
+    last_amended    the section's last_amended in the corpus now
+    last_amended_seen  the same in `then` (the approved corpus); '' when there
+                    is no baseline or the section was not in it
+    amended         last_amended is later than last_amended_seen
+    amended_by      the last citation in the section's history line
+    subsections     for an amended row: the quoted subsections whose stated
+                    maximum changed ([] = the amendment left it alone)
+    maxima_before / maxima_after  what those subsections stated then / now
+    provisions_added / provisions_removed  quoted subsections that exist
+                    only now / only then (created or repealed: re-read)
+    """
+    out = []
+    for r in rows:
+        act, sec = r["act"], r["section"]
+        text = now.text(act, sec)
+        rep = dict(r, status="ok", reason="", last_amended=now.last_amended(act, sec),
+                   last_amended_seen="", amended=False, amended_by="",
+                   subsections=[], maxima_before=[], maxima_after=[],
+                   provisions_added=[], provisions_removed=[])
+        if not text:
+            rep.update(status="broken", reason="section not in corpus")
+        elif not phrase_in(r["contains"], text):
+            rep.update(status="broken", reason="phrase not in the section")
+        if then is not None and then.section(act, sec):
+            seen = then.last_amended(act, sec)
+            rep["last_amended_seen"] = seen
+            if seen and rep["last_amended"] > seen:
+                labels, before, after, added, removed = maximum_change(
+                    r["contains"], then.raw_text(act, sec), now.raw_text(act, sec))
+                rep.update(amended=True, amended_by=now.amended_by(act, sec),
+                           subsections=labels, maxima_before=before, maxima_after=after,
+                           provisions_added=added, provisions_removed=removed)
+                if rep["status"] == "ok":
+                    rep["status"] = "amended"
+        out.append(rep)
+    return out
+
+
+def _day_before(iso):
+    return (datetime.date.fromisoformat(iso) - datetime.timedelta(days=1)).isoformat()
+
+
+def _years(m):
+    return "life" if m is not None and m >= 99 else f"{m} years"
+
+
+def pit_url(act, section, start):
+    """The Justice Laws point-in-time page of the version that began on
+    `start` (a candidate address, not fetched)."""
+    kind = "regulations" if act.upper().startswith(("SOR-", "C.R.C.", "CRC")) else "acts"
+    return (f"https://laws-lois.justice.gc.ca/eng/{kind}/{act}/section-{section}-"
+            f"{start.replace('-', '')}.html")
+
+
+def draft_history(report, equivalency):
+    """DRAFT penalty_history rows for the amended rows whose quoted maximum
+    changed, and which no row the pairing carries already covers. A covering
+    row ends between the amendment the review saw and the new one; for a
+    pairing-level verify row any limb's row counts. Each report row that a
+    draft or an existing row answers gets `draft` or `covered_by`.
+
+    The proposed row is the maximum before the amendment until the day
+    before it, when the quoted subsections stated a single maximum before.
+    Otherwise max_years is None and the note says so. It is never applied
+    here.
+
+    Each draft also carries `cautions` for the reviewer: the pairing (or
+    limb) has no structured maximum, so a row cannot drive the branch by
+    itself; or the same section is quoted at another maximum in a provision
+    the amendment left alone, so a row at this level would misstate that
+    limb. `prose_names_date` says whether the curated prose already names
+    the amendment (only the structured row is missing)."""
+    from .screening import maxima_stated
+    pairings = {p.get("id"): p for p in (equivalency or {}).get("pairings") or []}
+    drafts, made = [], {}
+    for r in report:
+        if not (r.get("amended") and r.get("subsections") and r["file"] == EQUIVALENCY):
+            continue
+        p = pairings.get(r["owner"])
+        if p is None:
+            continue
+        until = _day_before(r["last_amended"])
+        covering = sorted({h.get("until") for h in p.get("penalty_history") or []
+                           if isinstance(h, dict)
+                           and (r["limb"] is None or h.get("limb") in (None, r["limb"]))
+                           and r["last_amended_seen"] <= (h.get("until") or "") < r["last_amended"]})
+        if covering:
+            r["covered_by"] = covering
+            continue
+        key = (r["owner"], r["limb"], until)
+        if key in made:
+            r["draft"] = made[key]
+            continue
+        before, after = r["maxima_before"], r["maxima_after"]
+        old = before[0] if len(before) == 1 else None
+        limb = next((l for l in p.get("limbs") or []
+                     if isinstance(l, dict) and l.get("key") == r["limb"]), None)
+        table_max = limb.get("max_years", p.get("max_years")) if limb else p.get("max_years")
+        labels = [l for l in r["subsections"] if l]
+        where = f"{r['act']} s. {r['section']}" + (
+            f" (subsection{'s' if len(labels) > 1 else ''} "
+            f"{', '.join(f'({l})' for l in labels)})" if labels else "")
+        if old is None:
+            note = (f"DRAFT: {where} as it stood before {r['last_amended']} "
+                    f"({r['amended_by'] or 'amendment'}); the maximum before it is not "
+                    f"determined mechanically (stated then: "
+                    f"{', '.join(_years(m) for m in before) or 'none'}) -- read the "
+                    f"point-in-time version and set max_years before accepting")
+        else:
+            note = (f"DRAFT: {where} before {r['last_amended']} "
+                    f"({r['amended_by'] or 'amendment'}): {_years(old)} "
+                    f"(now {', '.join(_years(m) for m in after) or 'none stated'}) -- "
+                    f"confirm against the point-in-time version before accepting")
+        row = {"until": until, "max_years": old, "note": note}
+        if r["limb"]:
+            row["limb"] = r["limb"]
+        level = f"limb {r['limb']!r}" if r["limb"] else "pairing"
+        # The other maxima the same owner quotes for this section: a
+        # provision the amendment left alone, which a row at this level
+        # would misstate (s. 286.1(1) stays at 5 years while (2) went 10 -> 14).
+        others = sorted({m for x in report
+                         if x["owner"] == r["owner"] and x["limb"] == r["limb"]
+                         and (x["act"], x["section"]) == (r["act"], r["section"])
+                         for m in maxima_stated(x["contains"])} - set(before) - set(after))
+        cautions = []
+        if table_max is None:
+            cautions.append(
+                f"the table gives this {level} no structured maximum (max_years null"
+                + (f", depends_on {p.get('depends_on')!r}" if p.get("depends_on") else "")
+                + "): a history row here cannot drive the branch on its own -- decide "
+                "whether the amended subsection needs a limb of its own")
+        if others:
+            cautions.append(
+                f"{r['act']} s. {r['section']} is also quoted at "
+                f"{', '.join(_years(m) for m in others)} in a provision the amendment left "
+                f"alone: a row on this {level} would apply to that provision too -- give "
+                f"the amended subsection{'s' if len(labels) > 1 else ''} "
+                f"{', '.join(f'({l})' for l in labels) or where} a limb of its own first")
+        prose = squash(" ".join([p.get("penalty") or "", p.get("analysis") or "",
+                                 p.get("branch") or ""] + list(p.get("caveats") or [])))
+        made[key] = len(drafts)
+        r["draft"] = made[key]
+        drafts.append({
+            "pairing": r["owner"], "limb": r["limb"], "act": r["act"],
+            "section": r["section"], "subsections": [l for l in r["subsections"] if l],
+            "amended": r["last_amended"], "amended_by": r["amended_by"],
+            "last_amended_seen": r["last_amended_seen"],
+            "maxima_before": before, "maxima_after": after,
+            "table_max_years": table_max,
+            "pit_url_to_read": (pit_url(r["act"], r["section"], r["last_amended_seen"])
+                                if r["last_amended_seen"] else ""),
+            "prose_names_date": bool(prose) and (until in prose or r["last_amended"] in prose),
+            "cautions": cautions,
+            "status": "DRAFT - for review", "proposed_row": row})
+    return drafts
+
+
+def write_drafts(drafts, report, baseline, today, draft_dir=None):
+    """Write the drafts, with the amended and broken report rows, to
+    <draft_dir>/penalty-history-draft-<today>.json, and return its path.
+    Nothing is written when there are no drafts. When the newest existing
+    draft file already holds the same drafts, that file's path is returned
+    instead, so a weekly check does not pile up copies."""
+    if not drafts:
+        return ""
+    draft_dir = Path(draft_dir) if draft_dir else DRAFTS
+    drafts = json.loads(json.dumps(drafts))
+    existing = sorted(draft_dir.glob(DRAFT_GLOB))
+    if existing:
+        try:
+            if json.loads(existing[-1].read_text(encoding="utf-8")).get("drafts") == drafts:
+                return str(existing[-1])
+        except ValueError:
+            pass
+    draft_dir.mkdir(parents=True, exist_ok=True)
+    path = draft_dir / f"penalty-history-draft-{today}.json"
+    payload = {
+        "generated": today,
+        "tool": "py -m canlex.refresh --check (refresh.check_curated_sections)",
+        "status": ("DRAFT research output for review. Nothing in this file is applied "
+                   "to data/curated/us_equivalency.json; an accepted row is added there "
+                   "separately after review."),
+        "baseline": baseline,
+        "method": ("A verify row is amended when its section's last_amended in the corpus "
+                   "now is later than in the corpus of the approved commit "
+                   "(last_amended_seen). A row is drafted when the maximum stated in a "
+                   "subsection the row quotes changed, and no penalty_history row of the "
+                   "pairing ends between the two amendment dates: the maximum before the "
+                   "amendment, until the day before it."),
+        "drafts": drafts,
+        "rows": [r for r in report if r.get("amended") or r["status"] == "broken"],
+    }
+    path.write_text(json.dumps(payload, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
+    return str(path)
+
+
+# ---- the approved baseline and one entry point ------------------------------------
+
+def approved_commit(root=ROOT):
+    """The approved commit of data/curated/review_state.json when git can read
+    it, else ''."""
+    try:
+        state = json.loads((Path(root) / "data" / "curated" / "review_state.json")
+                           .read_text(encoding="utf-8"))
+        commit = state["approved"]["commit"]
+        ok = subprocess.run(["git", "-C", str(root), "cat-file", "-e", f"{commit}^{{commit}}"],
+                            capture_output=True, timeout=60).returncode == 0
+    except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError):
+        return ""
+    return commit if ok else ""
+
+
+def check_sections(equivalency, transborder=None, now=None, then=None):
+    """check_rows over every verify row of the equivalency table and the
+    IRPR s. 19 table (optional: absent on the approved baseline)."""
+    rows = verify_rows(equivalency) + verify_rows(transborder, TRANSBORDER)
+    return check_rows(rows, now or Corpus(), then)
+
+
+def run_checks(curated_dir=None, root=ROOT, baseline=None, today=None,
+               draft_dir=None, write=True):
+    """The whole T4.3 check on a tree's curated tables and corpus:
+    {baseline, rows, drafts, draft_file, drift, error}. baseline None reads
+    the approved commit from review_state.json; '' checks broken phrases
+    only."""
+    today = today or datetime.date.today().isoformat()
+    curated_dir = Path(curated_dir) if curated_dir else Path(root) / "data" / "curated"
+    try:
+        eq = json.loads((curated_dir / EQUIVALENCY).read_text(encoding="utf-8"))
+        tb_path = curated_dir / TRANSBORDER
+        tb = json.loads(tb_path.read_text(encoding="utf-8")) if tb_path.exists() else None
+    except (OSError, ValueError) as exc:
+        return {"baseline": "", "rows": [], "drafts": [], "draft_file": "", "drift": [],
+                "error": f"{type(exc).__name__}: {exc}"}
+    base = approved_commit(root) if baseline is None else baseline
+    now = Corpus(None, root)
+    then = Corpus(base, root) if base else None
+    rows = check_sections(eq, tb, now, then)
+    drafts = draft_history(rows, eq)
+    drift = [list(d) for d in refresh_drift(eq, now.text, then.text)[0]] if then else []
+    path = write_drafts(drafts, rows, base, today, draft_dir) if write else ""
+    return {"baseline": base, "rows": rows, "drafts": drafts, "draft_file": path,
+            "drift": drift, "error": ""}
+
+
+def gate(result):
+    """(status, lines) for tools/predeploy.py: FAIL on a broken phrase (or a
+    table that cannot be read), WARN on an amended section, PASS otherwise."""
+    if result.get("error"):
+        return FAIL, [f"curated tables unreadable: {result['error']}"]
+    rows = result.get("rows") or []
+    broken = [r for r in rows if r["status"] == "broken"]
+    amended = [r for r in rows if r.get("amended")]
+    lines = [f"BROKEN {where(r)}: {r['act']} s. {r['section']} -- {r['reason']}: "
+             f"{r['contains']!r}" for r in broken]
+    for pid, act, sec, new in result.get("drift") or []:
+        lines.append(f"NOT QUOTED {pid}: {act} s. {sec} now states "
+                     f"{' and '.join(_years(m) for m in new)}")
+    for d in result.get("drafts") or []:
+        lines.append(f"DRAFT history row {d['pairing']}"
+                     + (f" limb {d['limb']}" if d["limb"] else "")
+                     + f": {d['act']} s. {d['section']} until {d['proposed_row']['until']}")
+    secs = sorted({(r["act"], r["section"]) for r in amended})
+    if secs:
+        lines.append(f"amended since the approved corpus: {len(secs)} verify section(s): "
+                     + ", ".join(f"{a} s. {s}" for a, s in secs))
+    if result.get("draft_file"):
+        lines.append(f"drafts: {result['draft_file']}")
+    if broken:
+        return FAIL, lines
+    if amended or result.get("drift"):
+        return WARN, lines
+    return PASS, lines or [f"{len(rows)} verify rows: every phrase is in its section"]

canlex/delegation.py

@@ -0,0 +1,509 @@
+"""Ingest instruments of delegation and designation under IRPA / IRPR.
+
+These instruments record which officer positions the Minister has delegated
+powers to, or designated for functions, under the Immigration and Refugee
+Protection Act and its Regulations. They are administrative instruments -- not
+enacted law, and not guidance -- so every chunk is tagged doc_type="delegation".
+
+Sources:
+  - the CBSA "Delegation of Authority and Designations of Officers ..."
+    instrument (HTML, cbsa-asfc.gc.ca);
+  - the IRCC "IL3 -- Instrument of Designation and Delegation" (PDF, canada.ca).
+
+    py -m canlex.delegation
+"""
+import io
+import re
+import sys
+
+from bs4 import BeautifulSoup
+from pypdf import PdfReader
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "delegation"
+OUT = PROCESSED_DIR / "delegation.json"
+
+# The CBSA IRPA delegation has been a moving target: the November 28, 2017
+# instrument was expressly superseded by a full restatement on May 8, 2023
+# (signed by Mendicino), which has itself been amended five times since. CBSA
+# does not publish a consolidated version, so the current effective state is the
+# 2023 restatement read together with its later amendments; we ingest each one
+# as a separate "act" so a user (or the LLM) sees the base item and any
+# amendments that touch it side-by-side in retrieval.
+_CBSA_DELEG_URL = ("https://www.cbsa-asfc.gc.ca/agency-agence/actreg-loireg/"
+                   "delegation/")
+_CBSA_DELEG_NAME = ("Delegation of Authority and Designations of Officers by "
+                    "the Minister of Public Safety and Emergency Preparedness "
+                    "under the Immigration and Refugee Protection Act and the "
+                    "Immigration and Refugee Protection Regulations")
+
+SOURCES = {
+    # The 2023 restatement -- the current base instrument.
+    "cbsa-2023-05": {
+        "code": "cbsa-2023-05",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-2023-05",
+        "act_short": "CBSA Deleg 2023-05-08",
+        "act_name": _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2023-05-08-eng.html",
+        "effective": "2023-05-08",
+    },
+    # Amendments to the 2023 restatement, in chronological order.
+    "cbsa-2023-09": {
+        "code": "cbsa-2023-09",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-AMEND-2023-09-08",
+        "act_short": "CBSA Deleg Amend 2023-09-08",
+        "act_name": "Amendment to the " + _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2023-09-08-eng.html",
+        "effective": "2023-09-08",
+    },
+    "cbsa-2023-11": {
+        "code": "cbsa-2023-11",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-AMEND-2023-11-17",
+        "act_short": "CBSA Deleg Amend 2023-11-17",
+        "act_name": "Amendment to the " + _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2023-11-17-eng.html",
+        "effective": "2023-11-17",
+    },
+    "cbsa-2024-03-05": {
+        "code": "cbsa-2024-03-05",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-AMEND-2024-03-05",
+        "act_short": "CBSA Deleg Amend 2024-03-05",
+        "act_name": "Amendment to the " + _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2024-03-05-eng.html",
+        "effective": "2024-03-05",
+    },
+    "cbsa-2024-03-15": {
+        "code": "cbsa-2024-03-15",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-AMEND-2024-03-15",
+        "act_short": "CBSA Deleg Amend 2024-03-15",
+        "act_name": "Amendment to the " + _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2024-03-15-eng.html",
+        "effective": "2024-03-15",
+    },
+    "cbsa-2025-07": {
+        "code": "cbsa-2025-07",
+        "kind": "html-cbsa",
+        "act_code": "CBSA-IRPA-DELEG-AMEND-2025-07-10",
+        "act_short": "CBSA Deleg Amend 2025-07-10",
+        "act_name": "Amendment to the " + _CBSA_DELEG_NAME,
+        "url": _CBSA_DELEG_URL + "irpa-lipr-2025-07-10-eng.html",
+        "effective": "2025-07-10",
+    },
+    # Separate authority: a peace-officer designation under IRPA s. 138(1).
+    # Narrative prose, not a Schedule table -- needs its own parser.
+    "cbsa-peaceofficer": {
+        "code": "cbsa-peaceofficer",
+        "kind": "html-cbsa-narrative",
+        "act_code": "CBSA-IRPA-PEACEOFF-2022-08",
+        "act_short": "CBSA Peace Officer Auth 2022-08-18",
+        "act_name": ("Authorization to have the Authority and Powers of a "
+                     "Peace Officer under the Immigration and Refugee "
+                     "Protection Act (subsection 138(1))"),
+        "url": _CBSA_DELEG_URL + "desig/po-ag_2022-08-eng.html",
+        "effective": "2022-08-18",
+    },
+    "ircc": {
+        "code": "ircc",
+        "kind": "pdf-ircc",
+        "act_code": "IRCC-IL3-DELEG",
+        "act_short": "IRCC IL3",
+        "act_name": ("IL3 — Instrument of Designation and Delegation, "
+                     "Immigration and Refugee Protection Act and Regulations"),
+        "url": ("https://www.canada.ca/content/dam/ircc/migration/ircc/english/"
+                "resources/manuals/il/il3-eng.pdf"),
+    },
+}
+
+
+def _normalize_refs(text):
+    """Expand the instruments' provision shorthand so the section numbers are
+    searchable as tokens: 'A55(1)' -> 'IRPA 55(1)', 'R39' -> 'IRPR 39'. Both
+    instruments write 'A' for the Act and 'R' for the Regulations."""
+    text = re.sub(r"\bA(?=\d)", "IRPA ", text)
+    text = re.sub(r"\bR(?=\d)", "IRPR ", text)
+    return _norm(text)
+
+
+def _fetch(url, dest, powershell=False, force=False):
+    """Fetch a page or file, caching the raw bytes under data/raw/delegation.
+    canada.ca blocks Python's HTTP client, so its PDF is fetched via PowerShell.
+    (refresh.py also calls this for its IL3 version check.)"""
+    return fetch_cached(url, dest, powershell=powershell, force=force)
+
+
+# --- CBSA instrument (HTML) ---------------------------------------------------
+
+def _delegates(cell):
+    """Flatten a 'Delegates / Designated officials' table cell into readable
+    lines. The cell groups officer positions under an organisation header
+    (<p class="h4">: CBSA, RCMP, IRCC) and an optional sub-heading
+    (<p class="h5">: a region or a headquarters branch), each followed by a
+    <ul> of position titles."""
+    lines, org, sub, org_used = [], "", "", False
+    for el in cell.find_all(["p", "ul"], recursive=False):
+        if el.name == "p":
+            classes = " ".join(el.get("class") or [])
+            text = _norm(el.get_text())
+            if not text:
+                continue
+            if "h4" in classes:
+                if org and not org_used:   # a previous org with no list of its own
+                    lines.append(org)
+                org, sub, org_used = text.rstrip(": "), "", False
+            elif "h5" in classes:
+                sub = text
+            else:
+                lines.append(text)         # a free-standing note
+        else:                              # <ul> of position titles
+            positions = "; ".join(
+                t for t in (_norm(li.get_text())
+                            for li in el.find_all("li", recursive=False)) if t)
+            if positions:
+                label = f"{org} — {sub}" if sub else org
+                lines.append(f"{label}: {positions}" if label else positions)
+                org_used = True
+    if org and not org_used:               # a trailing org with no list
+        lines.append(org)
+    return "\n".join(lines)
+
+
+def parse_cbsa(html, src):
+    """Parse the CBSA delegation instrument into one chunk per Schedule item,
+    plus one chunk for the preamble."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    for sup in main.find_all("sup"):       # drop footnote-reference superscripts
+        sup.decompose()
+
+    # The effective date comes from SOURCES, not the first <time> in <main>:
+    # amendment pages quote the base instrument's date ("signed on May 8, 2023")
+    # in their preamble, so the first <time> on an amendment page is the base
+    # instrument's date, not the amendment's own.
+    date = src["effective"]
+
+    chunks = []
+
+    # Preamble: the paragraphs between the title and the Schedule, which set out
+    # the tiers of delegation and designation and how the columns are read.
+    schedule = main.find("h2", id="sch")
+    if schedule:
+        paras = [_norm(p.get_text())
+                 for p in reversed(schedule.find_previous_siblings("p"))]
+        body = "\n".join(p for p in paras if p)
+        if body:
+            chunks.append({
+                "id": f"delegation-{src['code']}-preamble",
+                "doc_type": "delegation",
+                "act_code": src["act_code"],
+                "act_short": src["act_short"],
+                "act_name": src["act_name"],
+                "section": "",
+                "marginal_note": "Tiers of delegation and designation",
+                "part": "",
+                "division": "",
+                "heading": "Instrument of delegation and designation under IRPA",
+                "text": body,
+                "history": "",
+                "last_amended": "",
+                "current_to": date,
+                "citation": f"{src['act_short']} — Preamble",
+                "source_url": src["url"],
+            })
+
+    # One chunk per Schedule item. Two row shapes are accepted:
+    #   (a) four <td> cells: Item | Refs | Power | Delegates (the 2023-05
+    #       restatement and the 2023-09, 2023-11, 2024-03 amendments).
+    #   (b) one <th> + two or three <td> cells: the <th> carries the item
+    #       number and the <td>s carry Refs | Power [| Delegates]. The
+    #       2025-07-10 amendment uses this layout, and may omit the Delegates
+    #       column when an amendment changes only references or descriptions.
+    # Each topical <h3>, if present, names the schedule section the table belongs to.
+    for table in main.find_all("table", class_="table-bordered"):
+        h3 = table.find_previous_sibling("h3")
+        section_name = _norm(h3.get_text()) if h3 else ""
+        for tr in table.find_all("tr"):
+            th_cells = tr.find_all("th", recursive=False)
+            td_cells = tr.find_all("td", recursive=False)
+            if not th_cells and len(td_cells) == 4:
+                item_cell, refs_cell, power_cell, deleg_cell = td_cells
+            elif len(th_cells) == 1 and len(td_cells) in (2, 3):
+                item_cell, refs_cell, power_cell = th_cells[0], td_cells[0], td_cells[1]
+                deleg_cell = td_cells[2] if len(td_cells) == 3 else None
+            else:
+                continue                   # header row or a stray row
+            item_no = _norm(item_cell.get_text()).rstrip(".")
+            refs = _normalize_refs(_norm(refs_cell.get_text()))
+            power = " ".join(_norm(p.get_text())
+                             for p in power_cell.find_all("p")) \
+                or _norm(power_cell.get_text())
+            delegates = _delegates(deleg_cell) if deleg_cell is not None else ""
+            if not item_no or not (power or refs):
+                continue
+            text = power
+            if refs:
+                text += f"\n\nProvisions (IRPA / IRPR): {refs}."
+            if delegates:
+                text += "\n\nDelegated / designated to:\n" + delegates
+            chunks.append({
+                "id": f"delegation-{src['code']}-{item_no}",
+                "doc_type": "delegation",
+                "act_code": src["act_code"],
+                "act_short": src["act_short"],
+                "act_name": src["act_name"],
+                "section": "",
+                "marginal_note": refs or f"Item {item_no}",
+                "part": section_name,
+                "division": "",
+                "heading": section_name,
+                "text": text,
+                "history": "",
+                "last_amended": "",
+                "current_to": date,
+                "citation": f"{src['act_short']}, Item {item_no}",
+                "source_url": src["url"],
+            })
+    return chunks
+
+
+# --- CBSA narrative-prose instrument (e.g. the peace-officer designation) -----
+
+def parse_cbsa_narrative(html, src):
+    """Parse a narrative-prose CBSA designation instrument into a single chunk.
+
+    Used for the peace-officer authorization under IRPA s. 138(1) -- plain prose
+    listing 18 designated officer positions, not a four-column Schedule table,
+    so parse_cbsa's table walker would yield nothing. The whole operative text
+    is a few hundred words, well within a single chunk."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    for sup in main.find_all("sup"):
+        sup.decompose()
+
+    date = src["effective"]
+
+    # Skip the breadcrumb/title/footer chrome -- only paragraphs and lists in
+    # <main> that carry real content. Lists are rendered as "; "-joined items.
+    parts = []
+    for el in main.find_all(["p", "ul", "ol"]):
+        if el.find_parent(["ul", "ol"]):
+            continue   # nested lists are picked up by their parent
+        if el.name in ("ul", "ol"):
+            items = [_norm(li.get_text()) for li in el.find_all("li")]
+            joined = "; ".join(t for t in items if t)
+            if joined:
+                parts.append(joined)
+        else:
+            text = _norm(el.get_text())
+            if text:
+                parts.append(text)
+    body = "\n".join(parts)
+    if not body:
+        return []
+
+    return [{
+        "id": f"delegation-{src['code']}",
+        "doc_type": "delegation",
+        "act_code": src["act_code"],
+        "act_short": src["act_short"],
+        "act_name": src["act_name"],
+        "section": "",
+        "marginal_note": "Peace-officer authorization — IRPA s. 138(1)",
+        "part": "",
+        "division": "",
+        "heading": src["act_name"],
+        "text": body,
+        "history": "",
+        "last_amended": "",
+        "current_to": date,
+        "citation": src["act_short"],
+        "source_url": src["url"],
+    }]
+
+
+# --- IRCC IL3 instrument (PDF) ------------------------------------------------
+
+# A topical part heading -- a line in full upper case (MINISTERIAL INSTRUCTIONS,
+# MEDICAL, MISCELLANEOUS). Organisation acronyms (CI, CBSA, RCMP) are shorter
+# than the 6-character floor and so are not mistaken for headings.
+_IL3_PART = re.compile(r"[A-Z][A-Z &/,()'.\-]{5,}")
+# An item opens "<n>. <A/R provision>" -- the number must be followed by a
+# provision reference, which rejects ordinary numbered prose.
+_IL3_ITEM = re.compile(r"(\d+)\.\s+(?=[AR]\d)")
+# The word that introduces an item's power description, after the provisions.
+_IL3_POWER = re.compile(r"(?:Delegation|Designation)\s*[-–—]\s*")
+
+
+def _pdf_pages(pdf_bytes):
+    """Extract each page's text, dropping the printed page number that pypdf
+    emits as the page's first line (roman in the front matter, arabic later)."""
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    pages = []
+    for page in reader.pages:
+        lines = (page.extract_text() or "").split("\n")
+        if lines and re.fullmatch(r"\s*[ivxlcdm\d]+\s*", lines[0] or "", re.I):
+            lines = lines[1:]
+        pages.append("\n".join(lines))
+    return pages
+
+
+def parse_ircc(pdf_bytes, src):
+    """Parse the IRCC IL3 instrument into one chunk per Schedule item, plus a
+    preamble chunk. The PDF flattens the four-column table into a linear text
+    stream, so each item runs from its numbered marker to the next; the power
+    description and the delegated positions are kept together within the item."""
+    pages = _pdf_pages(pdf_bytes)
+    if not pages:
+        return []
+    version_match = re.search(r"(?:Spring|Summer|Fall|Winter)\s+\d{4}", pages[0])
+    version = version_match.group(0) if version_match else ""
+
+    chunks = []
+    preamble = _norm(pages[1]) if len(pages) > 1 else ""
+    if preamble:
+        chunks.append({
+            "id": f"delegation-{src['code']}-preamble",
+            "doc_type": "delegation",
+            "act_code": src["act_code"],
+            "act_short": src["act_short"],
+            "act_name": src["act_name"],
+            "section": "",
+            "marginal_note": "Preamble — designation and delegation under IRPA s. 6",
+            "part": "",
+            "division": "",
+            "heading": "Instrument of designation and delegation under IRPA",
+            "text": preamble,
+            "history": "",
+            "last_amended": "",
+            "current_to": version,
+            "citation": f"{src['act_short']} — Preamble",
+            "source_url": src["url"],
+        })
+
+    # Walk every line: an all-caps line is a topical part heading; a line that
+    # opens "<n>. <A/R provision>" starts a new item. Lines before the first
+    # item (the preamble, definitions and table of contents) are skipped.
+    items, part = [], ""
+    cur_no = cur_part = None
+    cur_lines = []
+    for line in "\n".join(pages).split("\n"):
+        s = _norm(line)
+        if not s:
+            continue
+        if _IL3_PART.fullmatch(s):
+            part = s.title()
+            continue
+        m = _IL3_ITEM.match(s)
+        if m:
+            if cur_no is not None:
+                items.append((cur_part, cur_no, cur_lines))
+            cur_no, cur_part, cur_lines = m.group(1), part, [s]
+        elif cur_no is not None:
+            cur_lines.append(s)
+    if cur_no is not None:
+        items.append((cur_part, cur_no, cur_lines))
+
+    for n, (item_part, item_no, lines) in enumerate(items, start=1):
+        body = re.sub(r"^\d+\.\s*", "", "\n".join(lines)).strip()
+        if not body:
+            continue
+        power = _IL3_POWER.search(body)
+        refs = _normalize_refs(body[:power.start()]) if power else ""
+        chunks.append({
+            "id": f"delegation-{src['code']}-{n}",
+            "doc_type": "delegation",
+            "act_code": src["act_code"],
+            "act_short": src["act_short"],
+            "act_name": src["act_name"],
+            "section": "",
+            "marginal_note": refs or f"Item {item_no}",
+            "part": item_part or "",
+            "division": "",
+            "heading": item_part or "",
+            "text": _normalize_refs(body),
+            "history": "",
+            "last_amended": "",
+            "current_to": version,
+            "citation": (f"{src['act_short']} — {item_part}, Item {item_no}"
+                         if item_part else f"{src['act_short']}, Item {item_no}"),
+            "source_url": src["url"],
+        })
+    return chunks
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of instruments this run could not build.
+
+    Ingestion is a full rebuild, so an instrument that fails its fetch -- or
+    that parses to nothing because CBSA changed the page's shape again -- would
+    silently vanish from the corpus. An instrument withdrawn on purpose is
+    removed from SOURCES instead, and so never lands in `failures`, which is
+    why preservation is restricted to the codes that actually failed here.
+    act_code is the per-instrument identity every parser stamps on its chunks.
+    Returns (chunks, preserved act_codes).
+    """
+    failed = {act_code for act_code, _why in failures}
+    return preserve_dropped(chunks, stored, key=lambda c: c["act_code"],
+                            only=failed)
+
+
+def build(allow_shrink=False, force=False):
+    """Fetch, parse and chunk every delegation instrument into delegation.json."""
+    all_chunks, failures = [], []
+    for src in SOURCES.values():
+        print(f"Ingesting {src['act_short']} ...")
+        try:
+            if src["kind"] == "html-cbsa":
+                html = _fetch(src["url"], RAW / f"{src['code']}.html",
+                              force=force)
+                chunks = parse_cbsa(html, src)
+            elif src["kind"] == "html-cbsa-narrative":
+                html = _fetch(src["url"], RAW / f"{src['code']}.html",
+                              force=force)
+                chunks = parse_cbsa_narrative(html, src)
+            elif src["kind"] == "pdf-ircc":
+                pdf = _fetch(src["url"], RAW / f"{src['code']}.pdf",
+                             powershell=True, force=force)
+                chunks = parse_ircc(pdf, src)
+            else:
+                chunks = []
+        except Exception as exc:
+            print(f"  !! {src['act_short']}: {type(exc).__name__}: {exc}")
+            failures.append((src["act_code"], f"{type(exc).__name__}: {exc}"))
+            continue
+        if not chunks:
+            failures.append((src["act_code"], "no content parsed"))
+        all_chunks.extend(chunks)
+        print(f"  {len(chunks)} chunks")
+    uniquify_ids(all_chunks)
+
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} instrument(s): "
+              f"{', '.join(preserved)}")
+    if not write_corpus(OUT, all_chunks, indent=1, allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} delegation chunks from {len(SOURCES)} "
+          f"instrument(s) -> {OUT}")
+    return True
+
+
+def main():
+    sys.exit(0 if build(allow_shrink="--allow-shrink" in sys.argv,
+                        force="--force" in sys.argv) else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/directive.py

@@ -0,0 +1,298 @@
+"""Ingest National Joint Council directives (HTML) into section-level chunks.
+
+NJC directives are negotiated by employer and bargaining-agent representatives;
+their provisions form part of collective agreements (and the rate tables in
+their appendices apply too). Chunks are tagged doc_type="directive".
+"""
+import re
+import sys
+
+from bs4 import BeautifulSoup
+
+from ._common import (MONTHS, fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, write_corpus)
+from .config import RAW_DIR, PROCESSED_DIR
+
+INDEX_URL = "https://www.njc-cnm.gc.ca/directive/en"
+BASE = "https://www.njc-cnm.gc.ca"
+DIRECTIVE_DIR = RAW_DIR / "directives"
+OUT_FILE = PROCESSED_DIR / "directives.json"
+_CONTENT_TAGS = {"p", "dl", "table", "h4", "h5", "h6", "blockquote"}
+_NUMBERED = re.compile(r"^(\d+(?:\.\d+)*)\s+(.+)")
+_CODE = re.compile(r"/directive/([^/]+)/")
+
+_PIECE_TARGET = 1800   # keep pieces under the embedder's ~2000-char window
+_EFFECTIVE = re.compile(
+    r"effective[^.<:]{0,20}?([A-Z][a-z]+)\s+(\d{1,2}),?\s+(\d{4})", re.I)
+
+
+def _split(text, target=_PIECE_TARGET):
+    """Split long text into ~target-sized pieces at line boundaries."""
+    return split_lines(text, target)
+
+
+def _page_effective_date(main):
+    """Latest 'effective <Month D, YYYY>' date on a directive page, as ISO.
+    Used as a currency fallback when the index gives no date (e.g. the Foreign
+    Service Directives, whose chunks otherwise render currency as 'n/a')."""
+    best = ""
+    for mon, day, year in _EFFECTIVE.findall(main.get_text(" ", strip=True)):
+        if mon.lower() in MONTHS:
+            iso = f"{int(year):04d}-{MONTHS[mon.lower()]:02d}-{int(day):02d}"
+            best = max(best, iso)
+    return best
+
+
+def _fetch(url, dest, force=False):
+    """Fetch via PowerShell's (.NET) HTTP stack -- the NJC site blocks
+    Python's HTTP client at the TLS layer."""
+    return fetch_cached(url, dest, force=force, powershell=True)
+
+
+def directive_links(force=False):
+    """Return [(url, title, date), ...] for the current NJC directives."""
+    html = _fetch(INDEX_URL, DIRECTIVE_DIR / "_index.html", force=force)
+    soup = BeautifulSoup(html, "html.parser")
+    ul = soup.find("ul", class_="directive-list")
+    if ul is None:
+        return []
+    out = []
+    for li in ul.find_all("li", recursive=False):
+        a = li.find("a", href=True)  # the first <a> is the current directive
+        if not a:
+            continue
+        url = a["href"] if a["href"].startswith("http") else BASE + a["href"]
+        span = li.find("span", class_="date")
+        out.append((url, _norm(a.get_text(" ", strip=True)),
+                    _norm(span.get_text()) if span else ""))
+    return out
+
+
+def _block_text(heading):
+    """Readable text from a heading up to the next h2/h3 (sections unwrapped)."""
+    lines = []
+    for sib in heading.find_next_siblings():
+        if sib.name in ("h2", "h3"):
+            break
+        if sib.name in ("ul", "ol"):
+            for li in sib.find_all("li", recursive=False):
+                item = _norm(li.get_text(" ", strip=True))
+                if item:
+                    lines.append(f"- {item}")
+        elif sib.name in _CONTENT_TAGS:
+            text = _norm(sib.get_text(" ", strip=True))
+            if text:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def parse_directive(html, url, title, date):
+    """Parse one NJC directive page into one chunk per h2/h3 section."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    for tag in main.find_all(["section", "div"]):
+        tag.unwrap()  # flatten so each heading and its content become siblings
+
+    code_match = _CODE.search(url)
+    code = code_match.group(1) if code_match else url
+    if not date:                       # recover a currency date from the page
+        date = _page_effective_date(main)
+    chunks = []
+    current_part = ""
+    sec_idx = 0
+    for h in main.find_all(["h2", "h3"]):
+        text = _norm(h.get_text(" ", strip=True))
+        if not text:
+            continue
+        if h.name == "h2":
+            current_part = text
+        body = _block_text(h)
+        if not body:
+            continue
+        numbered = _NUMBERED.match(text)
+        if numbered:
+            number, note = numbered.group(1), numbered.group(2).strip()
+            citation = f"{title}, s. {number}"
+        else:
+            # Un-numbered heading: keep the text in the note only, so the rendered
+            # header (citation + note) does not repeat it.
+            number, note = "", text
+            citation = title
+        sec_idx += 1
+        # Split a long section so each piece fits the embed window; pieces of one
+        # section share the base id 'directive-{code}-{sec_idx}' and take a
+        # '-p<k>' suffix, so the diversity cap treats them as one source.
+        pieces = _split(body)
+        for k, piece in enumerate(pieces, start=1):
+            base = f"directive-{code}-{sec_idx}"
+            chunk_id = base if len(pieces) == 1 else f"{base}-p{k}"
+            pnote = note if len(pieces) == 1 else f"{note} (part {k})"
+            chunks.append({
+                "id": chunk_id,
+                "doc_type": "directive",
+                "act_code": code,
+                "act_short": title,
+                "act_name": f"NJC {title}",
+                "section": number or text,
+                "marginal_note": pnote,
+                "part": current_part if h.name == "h3" else "",
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": "",
+                "current_to": date,
+                "citation": citation,
+                "source_url": url,
+            })
+    return chunks
+
+
+def _print_link(html):
+    """Find a 'Print Full Directive' / 'Print all FSDs' link on a TOC page
+    (or its French label on the /fr editions)."""
+    soup = BeautifulSoup(html, "html.parser")
+    for a in soup.find_all("a", href=True):
+        text = _norm(a.get_text(" ", strip=True)).lower()
+        if ("print full directive" in text or "print all fsd" in text
+                or "imprimer la directive" in text
+                or "imprimer toutes" in text or "imprimez toutes" in text):
+            return a["href"] if a["href"].startswith("http") else BASE + a["href"]
+    return None
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of directives this run could not parse.
+
+    Ingestion is a full rebuild, so a directive whose page 404s or is restyled
+    vanishes from the corpus entirely -- and a directive that is still in force
+    is better served from the copy we already had than not at all. Keyed on
+    act_code (the /directive/<code>/ slug), which survives the title rewordings
+    the NJC index goes through; restricted to the directives that actually
+    failed, so any the NJC has genuinely retired stay gone.
+    Returns (chunks, preserved directive codes).
+    """
+    failed = {code for code, _title, _why in failures}
+    return preserve_dropped(chunks, stored, key=lambda c: c["act_code"],
+                            only=failed)
+
+
+def ingest(force=False, limit=None, allow_shrink=False):
+    directives = directive_links(force=force)
+    if limit:
+        directives = directives[:limit]
+    print(f"Ingesting {len(directives)} NJC directives...")
+    all_chunks, failures = [], []
+    for url, title, date in directives:
+        code_match = _CODE.search(url)
+        code = code_match.group(1) if code_match else "x"
+        try:
+            html = _fetch(url, DIRECTIVE_DIR / f"{code}.html", force=force)
+            chunks = parse_directive(html, url, title, date)
+            if not chunks:
+                # Multi-page directive: the landing page is only a table of
+                # contents -- follow its "Print Full Directive" link.
+                print_url = _print_link(html)
+                if print_url:
+                    full = _fetch(print_url, DIRECTIVE_DIR / f"{code}-full.html",
+                                  force=force)
+                    chunks = parse_directive(full, url, title, date)
+            if chunks:
+                all_chunks.extend(chunks)
+                print(f"  {title}: {len(chunks)} chunks")
+            else:
+                failures.append((code, title, "no content parsed"))
+        except Exception as exc:
+            failures.append((code, title, f"{type(exc).__name__}: {exc}"))
+    if not limit:
+        all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                                stored_chunks(OUT_FILE))
+        if preserved:
+            print(f"  kept the last-good copy of {len(preserved)} unparseable "
+                  f"directive(s): {', '.join(preserved)}")
+    # Before the write, so a refusal does not bury why the run came up short.
+    for _code, title, why in failures:
+        print(f"  FAILED {title}: {why}")
+    # The guard applies to --limit runs too: a smoke test over two directives
+    # must not be able to write a two-directive corpus over the real one.
+    if not write_corpus(OUT_FILE, all_chunks, indent=2,
+                        allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} chunks from {len(directives) - len(failures)} "
+          f"directives -> {OUT_FILE.name}")
+    return True
+
+
+OUT_FILE_FR = OUT_FILE.with_name("directives-fr.json")
+
+
+def ingest_fr(force=False, limit=None, allow_shrink=False):
+    """The French editions of the NJC directives, mirroring ingest().
+
+    The NJC publishes each directive at .../directive/<code>/fr with the
+    same markup as /en, so the parser is reused as-is. The EN index supplies
+    the directive list and identity (act_short stays the English title so
+    act filters behave one way regardless of language); chunks get
+    lang='fr' and a -fr id infix, and language-routed retrieval serves them
+    only to French queries."""
+    directives = directive_links(force=force)
+    if limit:
+        directives = directives[:limit]
+    print(f"Ingesting {len(directives)} directives du CNM (français)...")
+    all_chunks, failures = [], []
+    for url, title, date in directives:
+        code_match = _CODE.search(url)
+        code = code_match.group(1) if code_match else "x"
+        fr_url = re.sub(r"/en$", "/fr", url)
+        try:
+            html = _fetch(fr_url, DIRECTIVE_DIR / f"{code}-fr.html",
+                          force=force)
+            chunks = parse_directive(html, fr_url, title, date)
+            if not chunks:
+                print_url = _print_link(html)
+                if print_url:
+                    full = _fetch(print_url,
+                                  DIRECTIVE_DIR / f"{code}-fr-full.html",
+                                  force=force)
+                    chunks = parse_directive(full, fr_url, title, date)
+            if chunks:
+                all_chunks.extend(chunks)
+                print(f"  {title} (fr): {len(chunks)} chunks")
+            else:
+                failures.append((code, title, "no content parsed"))
+        except Exception as exc:
+            failures.append((code, title, f"{type(exc).__name__}: {exc}"))
+    for c in all_chunks:
+        c["id"] = c["id"].replace("directive-", "directive-fr-", 1)
+        c["lang"] = "fr"
+    if not limit:
+        all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                                stored_chunks(OUT_FILE_FR))
+        if preserved:
+            print(f"  kept the last-good copy of {len(preserved)} "
+                  f"unparseable directive(s)")
+    for _code, title, why in failures:
+        print(f"  FAILED {title} (fr): {why}")
+    if not write_corpus(OUT_FILE_FR, all_chunks, indent=2,
+                        allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} chunks from "
+          f"{len(directives) - len(failures)} directives -> "
+          f"{OUT_FILE_FR.name}")
+    return True
+
+
+def main():
+    force = "--force" in sys.argv
+    limit = next((int(a.split("=", 1)[1]) for a in sys.argv[1:]
+                  if a.startswith("--limit=")), None)
+    runner = ingest_fr if "--fr" in sys.argv else ingest
+    ok = runner(force=force, limit=limit,
+                allow_shrink="--allow-shrink" in sys.argv)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/dmemo.py

@@ -0,0 +1,374 @@
+"""Ingest CBSA D-Memoranda (HTML) into structured, section-level chunks.
+
+D-Memoranda are CBSA's administrative guidance on how it applies the Customs Act
+and related law. They are persuasive, not binding -- every chunk is tagged
+doc_type="memorandum" so the rest of CanLex can keep them distinct from statute.
+"""
+import io
+import json
+import re
+import sys
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+from pypdf import PdfReader
+
+from ._common import (POLITE_UA, dehyphenate, document_words, fetch_cached,
+                      norm_ws as _norm, preserve_dropped, split_lines,
+                      stored_chunks, write_corpus)
+from .config import RAW_DIR, PROCESSED_DIR
+
+INDEX_URL = "https://www.cbsa-asfc.gc.ca/publications/dm-md/d1-d23-eng.html"
+# CBSA rebuilt the index on 2026-07-10: the page now ships an empty WET
+# DataTable and NO memo links at all -- the rows arrive from this JSON
+# datasource. Each row's NUMBER holds the anchor
+# ('<a href="/publications/dm-md/d1/d1-2-1-eng.html">D1-2-1</a>'), and
+# cancelled memos carry STATUS "Cancelled" (usually with the link stripped).
+# Cancelled memos are deliberately not ingested -- same rule the ENF manuals
+# follow: retired guidance must not surface as if it were current.
+DATASOURCE_URL = ("https://www.cbsa-asfc.gc.ca/publications/dm-md/"
+                  "dmemoranda-datasource-eng.json")
+DMEMO_DIR = RAW_DIR / "dmemos"
+OUT_FILE = PROCESSED_DIR / "dmemos.json"
+
+# <h2> headings that are page boilerplate rather than memo content.
+_SKIP_HEADINGS = {"contact us", "related links",
+                  "contactez-nous", "liens connexes"}
+# Bilingual: the French editions live at the same paths with -fra.html, so
+# one pattern serves both ingest passes.
+_MEMO_HREF = re.compile(r"/dm-md/d\d+/d[\d-]+-(?:eng|fra)\.html")
+_URL_NUMBER = re.compile(r"/(d\d+-[\d-]+)-(?:eng|fra)\.html")
+_DS_HREF = re.compile(r'href="([^"]+)"')
+DATASOURCE_URL_FR = DATASOURCE_URL.replace("-eng.json", "-fra.json")
+OUT_FILE_FR = PROCESSED_DIR / "dmemos-fr.json"
+
+
+def _fetch(url, dest, force=False):
+    return fetch_cached(url, dest, ua=POLITE_UA, force=force)
+
+
+def datasource_urls(raw):
+    """Active D-memo URLs from the index's JSON datasource. Pure, for tests."""
+    urls, seen = [], set()
+    for row in json.loads(raw).get("data", []):
+        if _norm(row.get("STATUS", "")).lower() in ("cancelled", "annulé"):
+            continue
+        match = _DS_HREF.search(row.get("NUMBER", ""))
+        if not match:
+            continue
+        full = urljoin(DATASOURCE_URL, match.group(1))
+        if full not in seen:
+            seen.add(full)
+            urls.append(full)
+    return urls
+
+
+def index_urls(html):
+    """D-memo URLs linked from the index page itself. Pure, for tests.
+
+    How the index worked before 2026-07 and how it would work again if CBSA
+    reverts; kept as the fallback for datasource_urls.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    urls, seen = [], set()
+    for a in soup.find_all("a", href=True):
+        if _MEMO_HREF.search(a["href"]):
+            full = urljoin(INDEX_URL, a["href"])
+            if full not in seen:
+                seen.add(full)
+                urls.append(full)
+    return urls
+
+
+def memo_urls(force=False):
+    """All individual active D-memo URLs, from the CBSA index's datasource,
+    falling back to links on the index page if the datasource is gone."""
+    try:
+        urls = datasource_urls(_fetch(DATASOURCE_URL,
+                                      DMEMO_DIR / "_datasource-eng.json",
+                                      force=force))
+    except Exception as exc:
+        print(f"  datasource unavailable ({type(exc).__name__}: {exc}); "
+              f"falling back to index page links")
+        urls = []
+    if not urls:
+        urls = index_urls(_fetch(INDEX_URL, DMEMO_DIR / "_index.html",
+                                 force=force))
+    return urls
+
+
+def _render_section(h2):
+    """Readable text from an <h2> up to the next <h2> (sections already unwrapped)."""
+    lines = []
+    for sib in h2.find_next_siblings():
+        if sib.name == "h2" or sib.get("id") == "wb-dtmd":
+            break
+        if sib.name in ("ul", "ol"):
+            for li in sib.find_all("li", recursive=False):
+                item = _norm(li.get_text(" ", strip=True))
+                if item:
+                    lines.append(f"- {item}")
+        else:
+            text = _norm(sib.get_text(" ", strip=True))
+            if text:
+                lines.append(text)
+    return "\n".join(lines)
+
+
+def parse_memo(html, url):
+    """Parse one D-memo HTML page into one chunk per <h2> content section."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    for section in main.find_all("section"):
+        section.unwrap()  # flatten so each <h2> and its content become siblings
+
+    match = _URL_NUMBER.search(url)
+    number = match.group(1).upper() if match else url
+    h1 = main.find("h1")
+    topic = ""
+    if h1:
+        # Pages vary: most carry the memo title in <h1><small>, others as plain
+        # "Memorandum DNN-N-N: Title" h1 text. Use the <small> if present, else
+        # the h1 text, and strip any leading memo-number prefix either way.
+        small = h1.find("small")
+        raw = (small.get_text(" ", strip=True) if small
+               else h1.get_text(" ", strip=True))
+        topic = re.sub(r"^Memorandum\s+D[\w-]+\s*[:–-]\s*", "",
+                       _norm(raw), flags=re.I)
+    dm = main.find("time", attrs={"property": "dateModified"})
+    date = _norm(dm.get("datetime") or dm.get_text()) if dm else ""
+
+    chunks = []
+    for h2 in main.find_all("h2"):
+        heading = _norm(h2.get_text(" ", strip=True))
+        if not heading or heading.lower() in _SKIP_HEADINGS:
+            continue
+        body = _render_section(h2)
+        if not body:
+            continue
+        # One chunk per h2 section. (Splitting long memos to the embed window
+        # was trialled and reverted: it tripled memo chunk-count and gave
+        # D-memoranda enough extra retrieval surface to out-rank the statutes
+        # and sibling memos they relate to -- a measured eval regression. The
+        # memo-tail recall gap is better addressed later by multi-vector
+        # embedding than by more competing chunks.)
+        chunks.append({
+            "id": f"dmemo-{number}-{len(chunks) + 1}",
+            "doc_type": "memorandum",
+            "act_code": "D-Memo",
+            "act_short": "D-Memo",
+            "act_name": "CBSA D-Memoranda",
+            "section": number,
+            "marginal_note": heading,
+            "part": topic,
+            "division": "",
+            "heading": "",
+            "text": body,
+            "history": "",
+            "last_amended": date,
+            "current_to": date,
+            "citation": f"Memorandum {number}",
+            "source_url": url,
+        })
+    return chunks
+
+
+def _pdf_clean(text):
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n[ \t]+", "\n", text)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _pdf_text(pdf_bytes):
+    try:
+        reader = PdfReader(io.BytesIO(pdf_bytes))
+        text = _pdf_clean("\n".join((p.extract_text() or "") for p in reader.pages))
+        # line-break hyphens closed up (keeping the hyphen unless the memo
+        # itself uses the joined word; 2026-09-22)
+        return dehyphenate(text, document_words([text]))
+    except Exception:
+        return ""
+
+
+def _split(text, target=3000):
+    """Split long PDF text into ~target-sized pieces at line boundaries."""
+    return split_lines(text, target)
+
+
+def parse_pdf_memo(html, url, force=False):
+    """Fallback for memos whose HTML page is only a stub linking to a PDF.
+    `force` re-downloads the PDF (a memo CBSA reposts as a revised PDF at
+    the same URL was invisible to --force until 2026-09-21)."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    pdf_href = next((a["href"] for a in main.find_all("a", href=True)
+                     if a["href"].lower().endswith(".pdf")), None)
+    if not pdf_href:
+        return []
+    pdf_url = urljoin(url, pdf_href)
+
+    match = _URL_NUMBER.search(url)
+    number = match.group(1).upper() if match else url
+    h1 = main.find("h1")
+    topic = _norm(h1.get_text(" ", strip=True)) if h1 else ""
+    topic = re.sub(r"^Memorandum\s+D[\w-]+\s*[:–-]\s*", "", topic, flags=re.I)
+    dm = main.find("time", attrs={"property": "dateModified"})
+    date = _norm(dm.get("datetime") or dm.get_text()) if dm else ""
+
+    pdf_bytes = _fetch(pdf_url, DMEMO_DIR / "pdf" / pdf_url.rsplit("/", 1)[-1],
+                       force=force)
+    text = _pdf_text(pdf_bytes)
+    if not text:
+        return []
+
+    parts = _split(text)
+    chunks = []
+    for i, part in enumerate(parts, 1):
+        label = topic or number
+        if len(parts) > 1:
+            label = f"{label} (part {i})"
+        chunks.append({
+            "id": f"dmemo-{number}-pdf{i}",
+            "doc_type": "memorandum",
+            "act_code": "D-Memo",
+            "act_short": "D-Memo",
+            "act_name": "CBSA D-Memoranda",
+            "section": number,
+            "marginal_note": label,
+            "part": topic,
+            "division": "",
+            "heading": "",
+            "text": part,
+            "history": "",
+            "last_amended": date,
+            "current_to": date,
+            "citation": f"Memorandum {number}",
+            "source_url": url,
+        })
+    return chunks
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of memos that could not be fetched.
+
+    Ingestion is a full rebuild, so anything that fails a fetch disappears from
+    the corpus. CBSA's rebuilt index (2026-07) lists several memos as Active
+    whose pages 404, and a transient network error looks the same -- in both
+    cases dropping guidance that is still in force is worse than serving the
+    copy we already had. Restricted to memos that actually failed, so the
+    cancelled ones (filtered out before fetching) are not resurrected.
+    Returns (chunks, preserved memo numbers).
+    """
+    failed = set()
+    for url, _why in failures:
+        match = _URL_NUMBER.search(url)
+        if match:
+            failed.add(match.group(1).upper())
+    return preserve_dropped(chunks, stored,
+                            key=lambda c: c["section"].upper(), only=failed)
+
+
+def ingest(force=False, limit=None, allow_shrink=False):
+    urls = memo_urls(force=force)
+    if limit:
+        urls = urls[:limit]
+    print(f"Ingesting {len(urls)} D-Memoranda...")
+    all_chunks, failures = [], []
+    for i, url in enumerate(urls, 1):
+        try:
+            html = _fetch(url, DMEMO_DIR / url.rsplit("/", 1)[-1], force=force)
+            chunks = parse_memo(html, url) or parse_pdf_memo(html, url, force=force)
+            if chunks:
+                all_chunks.extend(chunks)
+            else:
+                failures.append((url, "no content parsed"))
+        except Exception as exc:
+            failures.append((url, f"{type(exc).__name__}: {exc}"))
+        if i % 50 == 0:
+            print(f"  {i}/{len(urls)} ...")
+    if failures:
+        print(f"  {len(failures)} memos with no content / errors:")
+        for url, why in failures[:15]:
+            print(f"    - {url.rsplit('/', 1)[-1]}: {why}")
+
+    if not limit:
+        all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                                stored_chunks(OUT_FILE))
+        if preserved:
+            print(f"  kept the last-good copy of {len(preserved)} unfetchable "
+                  f"memo(s): {', '.join(preserved)}")
+    # The guard applies to --limit runs too: a smoke test over five memos must
+    # not be able to write a five-memo corpus over the real one.
+    if not write_corpus(OUT_FILE, all_chunks, allow_shrink=allow_shrink):
+        return False
+    print(f"  {len(all_chunks)} section-chunks from {len(urls) - len(failures)} memos "
+          f"-> {OUT_FILE.name}")
+    return True
+
+
+def ingest_fr(force=False, limit=None, allow_shrink=False):
+    """The French editions (Mémorandums D), mirroring ingest().
+
+    Same pages at -fra.html, same WET markup, same parser; the chunks are
+    post-processed with lang='fr', a dmemo-fr- id prefix and the French
+    citation so language-routed retrieval (index.query_lang) serves them
+    only to French queries."""
+    try:
+        urls = datasource_urls(_fetch(DATASOURCE_URL_FR,
+                                      DMEMO_DIR / "_datasource-fra.json",
+                                      force=force))
+    except Exception as exc:
+        print(f"  FR datasource unavailable ({type(exc).__name__}: {exc})")
+        urls = []
+    if limit:
+        urls = urls[:limit]
+    print(f"Ingesting {len(urls)} Mémorandums D (français)...")
+    all_chunks, failures = [], []
+    for i, url in enumerate(urls, 1):
+        try:
+            html = _fetch(url, DMEMO_DIR / url.rsplit("/", 1)[-1], force=force)
+            chunks = parse_memo(html, url) or parse_pdf_memo(html, url, force=force)
+            if chunks:
+                all_chunks.extend(chunks)
+            else:
+                failures.append((url, "no content parsed"))
+        except Exception as exc:
+            failures.append((url, f"{type(exc).__name__}: {exc}"))
+        if i % 50 == 0:
+            print(f"  {i}/{len(urls)} ...")
+    for c in all_chunks:
+        c["id"] = c["id"].replace("dmemo-", "dmemo-fr-", 1)
+        c["lang"] = "fr"
+        c["citation"] = c["citation"].replace("Memorandum", "Mémorandum", 1)
+    if failures:
+        print(f"  {len(failures)} mémorandums with no content / errors "
+              f"(first: {failures[0][0].rsplit('/', 1)[-1]})")
+    if not limit:
+        all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                                stored_chunks(OUT_FILE_FR))
+        if preserved:
+            print(f"  kept the last-good copy of {len(preserved)} "
+                  f"unfetchable mémorandum(s)")
+    if not write_corpus(OUT_FILE_FR, all_chunks, allow_shrink=allow_shrink):
+        return False
+    print(f"  {len(all_chunks)} section-chunks from "
+          f"{len(urls) - len(failures)} mémorandums -> {OUT_FILE_FR.name}")
+    return True
+
+
+def main():
+    force = "--force" in sys.argv
+    limit = next((int(a.split("=", 1)[1]) for a in sys.argv[1:]
+                  if a.startswith("--limit=")), None)
+    runner = ingest_fr if "--fr" in sys.argv else ingest
+    ok = runner(force=force, limit=limit,
+                allow_shrink="--allow-shrink" in sys.argv)
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/embed.py

@@ -0,0 +1,267 @@
+"""Build semantic embeddings for ingested chunks (local, key-free).
+
+Uses BAAI's bge-small-en-v1.5 sentence-embedding model as ONNX, run on CPU via
+onnxruntime -- no API key. A transformer embedding has far stronger retrieval
+recall than a static one: it can connect a natural-language question to a
+provision even when the two share few exact words.
+
+Incremental by default: embeddings.npz stores a content hash per row, and a
+rebuild re-encodes only the chunks whose embed text changed (or is new),
+reusing every other row. A weekly consolidation refresh that touches a few
+hundred sections therefore takes seconds, not the ~40 minutes of a full
+pass over ~55k windows. `--full` forces the full pass (use it after changing
+the model, the pooling, the prefixes or embed_texts itself -- the hash
+covers the text, not the recipe).
+
+    python -m canlex.embed [--full]
+    python -m canlex.embed --check    # exit 1 unless the store is current
+"""
+import hashlib
+import json
+import os
+import sys
+
+import numpy as np
+from tokenizers import Tokenizer
+
+from .config import PROCESSED_DIR
+
+# The embedding model is env-configurable so an alternative (e.g. a multilingual
+# model for French support) can be trialled without code edits; the defaults
+# reproduce the production bge-small-en-v1.5 setup exactly. A model swap means
+# matching three things to that model's training: the HF repo, the pooling
+# ('cls' for BGE, 'mean' for E5/MiniLM), and the query/passage prefixes (BGE
+# prefixes only the query; E5 prefixes both with 'query: ' / 'passage: ').
+EMB_REPO = os.environ.get("CANLEX_EMB_REPO", "Xenova/bge-small-en-v1.5")
+_POOLING = os.environ.get("CANLEX_EMB_POOLING", "cls").lower()
+_QUERY_PREFIX = os.environ.get(
+    "CANLEX_EMB_QUERY_PREFIX",
+    "Represent this sentence for searching relevant passages: ")
+_PASSAGE_PREFIX = os.environ.get("CANLEX_EMB_PASSAGE_PREFIX", "")
+EMB_PATH = PROCESSED_DIR / "embeddings.npz"
+_MAX_TOKENS = 512
+_MAX_BODY = 2000   # cap embedded body text so long sections stay topically focused
+
+
+def load_chunks():
+    chunks = []
+    for path in sorted(PROCESSED_DIR.glob("*.json")):
+        chunks.extend(json.loads(path.read_text(encoding="utf-8")))
+    return chunks
+
+
+# A chunk longer than the embed window gets additional vectors for its tail,
+# up to this many per chunk (an 87K-char D-memo section would otherwise need
+# 40+; eight windows cover 16K chars, past which BM25 carries the tail).
+_MAX_WINDOWS = 8
+
+
+def embed_text(chunk):
+    """Compact, retrieval-focused representation of one section (head only)."""
+    # The section title is the strongest topical signal, so it is repeated to
+    # emphasise it. Title selection is doc_type-aware (see index.topical_title):
+    # a D-memo's marginal_note is a generic banner so its actual subject in
+    # 'part' is used; a case-law chunk's marginal_note is just the paragraph
+    # range so the case proposition in 'heading' is used.
+    from .index import topical_title
+    note = topical_title(chunk)
+    body = chunk["text"][:_MAX_BODY]
+    parts = [chunk["act_short"], note, note, chunk["heading"], body]
+    return " . ".join(p for p in parts if p)
+
+
+def embed_texts(chunk):
+    """One representation per embedding window of a chunk.
+
+    The first is the classic head representation; a chunk whose body exceeds
+    the embed window additionally gets tail windows (same topical-title
+    anchor, successive body slices), so a fact buried mid-memo is reachable
+    through semantic recall instead of BM25 alone. All windows share the
+    chunk's id -- the index scores a chunk by its best window."""
+    from .index import topical_title
+    texts = [embed_text(chunk)]
+    body = chunk["text"]
+    if len(body) > _MAX_BODY:
+        note = topical_title(chunk)
+        anchor = " . ".join(p for p in (chunk["act_short"], note) if p)
+        for start in range(_MAX_BODY, len(body), _MAX_BODY):
+            if len(texts) >= _MAX_WINDOWS:
+                break
+            texts.append(f"{anchor} . {body[start:start + _MAX_BODY]}")
+    return texts
+
+
+class Embedder:
+    """Local transformer sentence-embedder: bge-small-en-v1.5 as ONNX on CPU.
+
+    No API key; the model is downloaded once and cached. Produces L2-normalized
+    vectors, so a dot product between them is cosine similarity.
+    """
+
+    def __init__(self):
+        # Each file from the local cache first (rerank.hub_file); the
+        # quantized model is preferred, the full one the fallback.
+        from .rerank import hub_file, make_ort_session, make_run_options
+        model_path = None
+        for name in ("onnx/model_quantized.onnx", "onnx/model.onnx"):
+            try:
+                model_path = hub_file(EMB_REPO, name)
+                break
+            except Exception:
+                continue
+        if model_path is None:
+            raise RuntimeError(f"Could not download an ONNX model from {EMB_REPO}.")
+        tok_path = hub_file(EMB_REPO, "tokenizer.json")
+        self.session = make_ort_session(model_path)
+        self.run_options = make_run_options()
+        self.input_names = {i.name for i in self.session.get_inputs()}
+        self.tokenizer = Tokenizer.from_file(tok_path)
+        self.tokenizer.enable_truncation(max_length=_MAX_TOKENS)
+
+    def _run(self, texts):
+        """Tokenize, run the encoder, CLS-pool and L2-normalize one batch."""
+        encs = self.tokenizer.encode_batch(list(texts))
+        width = max(len(e.ids) for e in encs)
+        input_ids = np.zeros((len(encs), width), dtype=np.int64)
+        attention = np.zeros((len(encs), width), dtype=np.int64)
+        type_ids = np.zeros((len(encs), width), dtype=np.int64)
+        for row, enc in enumerate(encs):
+            n = len(enc.ids)
+            input_ids[row, :n] = enc.ids
+            attention[row, :n] = enc.attention_mask
+            type_ids[row, :n] = enc.type_ids
+        feed = {"input_ids": input_ids, "attention_mask": attention}
+        if "token_type_ids" in self.input_names:
+            feed["token_type_ids"] = type_ids
+        hidden = np.asarray(self.session.run(None, feed, self.run_options)[0],
+                            dtype=np.float32)
+        if hidden.ndim != 3:
+            pooled = hidden                                    # already pooled
+        elif _POOLING == "mean":                               # E5 / MiniLM
+            mask = attention[:, :, None].astype(np.float32)
+            pooled = (hidden * mask).sum(1) / np.maximum(mask.sum(1), 1e-9)
+        else:                                                  # BGE: CLS token
+            pooled = hidden[:, 0, :]
+        norms = np.linalg.norm(pooled, axis=1, keepdims=True)
+        return pooled / np.maximum(norms, 1e-9)
+
+    def encode(self, texts, batch_size=32):
+        """Return L2-normalized embeddings for passages, one row per text."""
+        texts = [_PASSAGE_PREFIX + t for t in texts]
+        if not texts:
+            return np.zeros((0, 384), dtype=np.float32)
+        rows = [self._run(texts[i:i + batch_size])
+                for i in range(0, len(texts), batch_size)]
+        return np.vstack(rows)
+
+    def encode_query(self, text):
+        """Return the L2-normalized embedding for one search query."""
+        return self._run([_QUERY_PREFIX + text])[0]
+
+
+def content_hash(texts):
+    """Hash of a chunk's embed windows -- the identity a stored row keeps."""
+    h = hashlib.sha1()
+    for t in texts:
+        h.update(t.encode("utf-8"))
+        h.update(b"\x00")
+    return h.hexdigest()
+
+
+def load_stored(path=None):
+    """{chunk id: (hash, rows)} from an existing npz, or {} when there is none
+    or it predates per-row hashes (then nothing can be trusted for reuse)."""
+    path = path or EMB_PATH
+    if not path.exists():
+        return {}
+    with np.load(path) as data:
+        if "hashes" not in data.files:
+            return {}
+        ids = data["ids"].tolist()
+        hashes = data["hashes"].tolist()
+        vectors = data["vectors"]
+        stored = {}
+        for row, (cid, h) in enumerate(zip(ids, hashes)):
+            entry = stored.setdefault(cid, (h, []))
+            if entry[0] == h:
+                entry[1].append(vectors[row])
+        return stored
+
+
+def plan(chunks, stored):
+    """Split chunks into (reusable, to_embed): a chunk is reusable when the
+    stored npz holds rows under its id with the same content hash. Pure, so
+    the reuse rule is testable without a model."""
+    reusable, to_embed = [], []
+    for c in chunks:
+        texts = embed_texts(c)
+        h = content_hash(texts)
+        entry = stored.get(c["id"])
+        if entry and entry[0] == h and len(entry[1]) == len(texts):
+            reusable.append((c["id"], h, entry[1]))
+        else:
+            to_embed.append((c["id"], h, texts))
+    return reusable, to_embed
+
+
+def currency(chunks=None, stored=None):
+    """(missing, stale): chunks with no stored vector at all, and chunks
+    whose stored vectors were embedded from other text (another content
+    hash or window count) -- the two halves of what an incremental build
+    would re-embed (plan's to_embed). (0, 0) means the store is current.
+    Defaults to the processed corpus and embeddings.npz. Audit follow-up
+    2026-09-21 (Missing 15): the only signal was a stderr line at load."""
+    chunks = load_chunks() if chunks is None else chunks
+    stored = load_stored() if stored is None else stored
+    _, to_embed = plan(chunks, stored)
+    missing = sum(1 for cid, _, _ in to_embed if cid not in stored)
+    return missing, len(to_embed) - missing
+
+
+def build(full=False, encoder=None):
+    chunks = load_chunks()
+    if not chunks:
+        print(f"No processed data in {PROCESSED_DIR}. Run 'canlex.ingest' first.")
+        return
+    stored = {} if full else load_stored()
+    reusable, to_embed = plan(chunks, stored)
+    n_windows = sum(len(t) for _, _, t in to_embed)
+    print(f"Embedding {len(to_embed)} of {len(chunks)} sections "
+          f"({n_windows} windows) with {EMB_REPO}; reusing "
+          f"{len(reusable)} unchanged ...")
+    fresh = {}
+    if to_embed:
+        encoder = encoder or Embedder()
+        texts = [t for _, _, ts in to_embed for t in ts]
+        vecs = encoder.encode(texts)
+        row = 0
+        for cid, h, ts in to_embed:
+            fresh[cid] = list(vecs[row:row + len(ts)])
+            row += len(ts)
+    by_id = {cid: (h, rows) for cid, h, rows in reusable}
+    by_id.update({cid: (h, fresh[cid]) for cid, h, _ in to_embed})
+    ids, hashes, vectors = [], [], []
+    for c in chunks:                       # chunk order, every window kept
+        h, rows = by_id[c["id"]]
+        for v in rows:
+            ids.append(c["id"])
+            hashes.append(h)
+            vectors.append(np.asarray(v, dtype=np.float32))
+    vectors = np.vstack(vectors) if vectors else np.zeros((0, 384), np.float32)
+    # float16 + compression: ~60% smaller on disk with no measurable
+    # retrieval cost (cosine ranks are insensitive to half-precision
+    # storage; the index upcasts to float32 at load). The npz ships in the
+    # Space repo via LFS, where every pushed version counts against the
+    # 1 GB storage cap -- size is a deploy constraint, not vanity.
+    np.savez_compressed(EMB_PATH, ids=np.array(ids), hashes=np.array(hashes),
+                        vectors=vectors.astype(np.float16))
+    print(f"  {vectors.shape[0]} vectors, dim {vectors.shape[1]} -> {EMB_PATH.name}")
+
+
+if __name__ == "__main__":
+    if "--check" in sys.argv:
+        # Exit 1 unless every processed chunk has a current vector.
+        n_missing, n_stale = currency()
+        print(f"embeddings.npz: {n_missing} chunk(s) missing, {n_stale} stale")
+        sys.exit(1 if n_missing or n_stale else 0)
+    build(full="--full" in sys.argv)

canlex/enf.py

@@ -0,0 +1,1023 @@
+"""Ingest the IRCC/CBSA ENF operational-manual chapters.
+
+The ENF chapters are the operational layer between the statute and the
+officer: ENF 4 (port-of-entry examinations), ENF 5 (writing A44(1) reports),
+ENF 6 (Minister's-delegate review), ENF 12 (search and seizure), ENF 20
+(detention) and their siblings are what an immigration-secondary BSO actually
+opens. The legacy PDFs live under the canada.ca manuals directory that
+delegation.py already fetches IL3 from (same PowerShell TLS workaround), and
+are parsed with pypdf, TOC-segmented by their internal numbered headings.
+
+Chapters are probed rather than hardcoded: the numbering has gaps and IRCC
+retires chapters, so anything that 404s is skipped with a note. doc_type is
+'memorandum' -- operational guidance, persuasive not binding, exactly like
+the D-memoranda.
+
+    python -m canlex.enf [--allow-shrink]
+"""
+import re
+import sys
+import urllib.error
+
+from ._common import (dehyphenate, document_words, fetch_cached,
+                      mark_soft_breaks, norm_ws as _norm, powershell_status,
+                      preserve_dropped, split_lines, stored_chunks, uniquify_ids,
+                      write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+BASE = ("https://www.canada.ca/content/dam/ircc/migration/ircc/english/"
+        "resources/manuals/enf/enf{n:02d}-eng.pdf")
+# The French editions, published beside the English ones (`--fr`). ENF 20's
+# French current edition is enf20a-fr.pdf (verified 2026-09-09); ENF 3's
+# standard French filename serves the same STALE 2015 edition as the English
+# one and no current French filename was found, so it is skipped rather than
+# ingested out of date; the ATIP chapters (2, 14) exist in English only.
+BASE_FR = ("https://www.canada.ca/content/dam/ircc/migration/ircc/francais/"
+           "ressources/guides/enf/enf{n:02d}-fra.pdf")
+_URL_OVERRIDES_FR = {
+    20: ("https://www.canada.ca/content/dam/ircc/migration/ircc/francais/"
+         "ressources/guides/enf/enf20a-fr.pdf"),
+}
+_FR_STALE = {3}
+RAW = RAW_DIR / "enf"
+_CHAPTERS = range(1, 41)      # probe; gaps and retired chapters just skip
+
+# Chapters whose current edition lives under a non-standard filename in the
+# same directory (verified 2026-07-22 against the OBM Active-manuals index).
+# ENF 3's old enf03-eng.pdf still returns 200 but is a STALE 2015-era copy;
+# ENF 20 (Detention) was renamed enf20a-en.pdf (Aug 2024 edition).
+_URL_OVERRIDES = {
+    3: ("https://www.canada.ca/content/dam/ircc/migration/ircc/english/"
+        "resources/manuals/enf/enf-3-admissibility-(en)-final.pdf"),
+    20: ("https://www.canada.ca/content/dam/ircc/migration/ircc/english/"
+         "resources/manuals/enf/enf20a-en.pdf"),
+}
+
+# Chapters IRCC withdrew from public distribution whose only public text is
+# an ATIP-released copy hosted by a third party (meurrensonimmigration.com,
+# verified live 2026-07-22). Ingested at the maintainer's decision
+# (2026-07-23) with an unmistakable UNOFFICIAL banner on every chunk: the
+# text may lag the internal current version, and nothing here is an official
+# publication. ENF 14/OP 19 is the operational manual behind criminal
+# rehabilitation -- directly relevant to canlex_rehabilitation.
+UNOFFICIAL = {
+    2: {"url": "https://meurrensonimmigration.com/wp-content/uploads/2023/06/"
+               "ENF2-3.pdf",
+        "title": "Evaluating Inadmissibility (unofficial ATIP copy)",
+        "note": "ATIP-released copy dated 2021-08-25; retired from canada.ca "
+                "in 2018 with no public successor"},
+    # ENF 14's ATIP release is a 56-page image scan with no text layer; its
+    # text comes from a Windows-OCR pass cached as enf14-ocr.txt (see the
+    # OCR fallback in build()). Regenerate the cache with pymupdf
+    # rasterization + Windows.Media.Ocr if the source PDF ever changes.
+    14: {"url": "https://meurrensonimmigration.com/wp-content/uploads/2025/03/"
+                "Enforcement-Manual-14-%E2%80%93-Rehabilitation.pdf",
+         "title": "Criminal Rehabilitation (ENF 14/OP 19, unofficial ATIP "
+                  "copy)",
+         "note": "withdrawn from public distribution ~2007; ATIP-released "
+                 "scan, text recovered by OCR (last internal change "
+                 "2008-04-21) -- OCR artifacts possible"},
+}
+# The structured status flag every chunk of those chapters carries
+# (`source_status`). Retrieval keys its ATIP-copy penalty on this field --
+# never on the act_name wording, which differs between the two chapters
+# ('(unofficial ATIP copy)' vs '(ENF 14/OP 19, unofficial ATIP copy)') and
+# so silently missed ENF 14 when a suffix test was tried (eval-diff draft
+# 2026-09-12). canlex/index.py ATIP_STATUS must equal this string.
+ATIP_STATUS = "unofficial-atip-copy"
+_UNOFFICIAL_BANNER = (
+    "[UNOFFICIAL ATIP COPY — this chapter was withdrawn from public "
+    "distribution; this text comes from an access-to-information release "
+    "hosted by a third party and may lag the current internal version. "
+    "Treat as historical guidance; verify against internal sources.] ")
+
+# Curated titles for the chapters; anything else gets a generic "ENF n".
+# NOT display-only: index.topical_title uses a memorandum's `part` (this
+# title) as its topical title, so it is weighted in BM25 and embedded. Every
+# entry for an ingested chapter was checked against the chapter's own cover
+# page on 2026-09-12 (tests/test_enf.py pins that when the PDFs are cached);
+# before that, ENF 14 (Criminal Rehabilitation) sat under a war-crimes title
+# and ENF 6, 17, 18, 21 and 34 carried other chapters' titles. 7, 9, 11, 13
+# and 27 are not ingested (absent upstream) and are unverified.
+TITLES = {
+    1: "Inadmissibility",
+    2: "Evaluating inadmissibility",
+    3: "Admissibility hearings and detention reviews",
+    4: "Port of entry examinations",
+    5: "Writing 44(1) reports",
+    6: "Review of reports under subsection A44(2)",
+    7: "Investigations and arrests",
+    8: "Deposits and guarantees",
+    9: "Judicial review",
+    10: "Removals",
+    11: "Verifying departure",
+    12: "Search, seizure, fingerprinting and photographing",
+    13: "Examining bank drafts and other negotiable instruments",
+    14: "Criminal rehabilitation",
+    15: "Obligations of transporters",
+    17: "Maritime procedures",
+    18: "Human or international rights violations",
+    19: "Appeals before the Immigration Appeal Division",
+    20: "Detention",
+    21: "Recovering missing, abducted and exploited children",
+    22: "Persons serving a sentence",
+    23: "Loss of permanent resident status",
+    24: "Ministerial interventions",
+    26: "Appeals at the Refugee Appeal Division",
+    27: "Permanent resident card",
+    28: "Ministerial opinions on danger to the public",
+    29: "Alternative means of examination programs",
+    34: "Alternatives to detention program",
+}
+
+_DATE = re.compile(r"\b(20\d{2}-\d{2}-\d{2})\b")
+# A manual heading line: "5. Departmental policy" / "5.1 Authority to ..."
+_HEADING = re.compile(r"^(\d{1,2}(?:\.\d{1,2})?)\.?\s+([A-ZÀ-Ý][^\n]{2,90})$")
+
+# --- Heading-candidate vetting --------------------------------------------
+# _HEADING alone accepted anything shaped "<number> <Capitalised text>", so
+# (measured on the 2026-09 corpus) 120 of ENF 2's 145 chunks sat under
+# "s. 1" titled by the ATIP page stamp ("1 A-2023-83511-000094"), ENF 1's
+# Saini note sat under "s. 64 (F.T.R. 311 (Fed. T.D.))" (a wrapped reporter
+# citation), ENF 19 opened with "s. 22 August 2024" and ENF 2 had
+# "s. 27 March 1992 until 10 October 1996." A candidate is rejected when its
+# text is one of those shapes, and then when its number does not continue the
+# chapter's numbering (numbering_run).
+# No \b before the "A": the stamp is sometimes glued to the column number
+# ("1A-2023-83511-000047").
+_ATIP_STAMP = re.compile(r"(?<![A-Za-z])A-\d{4}-\d+-\d+\b")
+_REPORTER = re.compile(
+    r"F\.T\.R\.|D\.L\.R\.|Imm\.\s?L\.R\.|\bN\.R\.|C\.R\.R\.|S\.C\.R\.|R\.C\.S\."
+    r"|C\.C\.C\.|Admin\.\s?L\.R\.|\(Fed\.|\bF\.C\.|\[\d{4}\]"
+    r"|\b(?:19|20)\d{2}\s+(?:FCA|FC|SCC|CAF|CF|CSC)\s+\d+")
+_MONTHS = ("January|February|March|April|May|June|July|August|September|"
+           "October|November|December|janvier|février|mars|avril|mai|juin|"
+           "juillet|août|septembre|octobre|novembre|décembre")
+# "August 2024", "March 1992 until ...", "May 3, 2000", "2021-08-25 ..."
+_DATE_START = re.compile(
+    rf"^(?:(?:{_MONTHS})\s+(?:\d{{1,2}}(?:er)?,?\s+)?\d{{4}}\b"
+    r"|\d{4}-\d{2}-\d{2}\b)", re.I)
+_LEADERS = re.compile(r"\.{4,}|…")            # a table-of-contents entry
+# A heading may legitimately end in an initialism ("... via the U.S.",
+# "... vers les É.-U."); any other final period marks a sentence.
+_INITIALISM_END = re.compile(r"(?<!\w)(?:[A-Za-zÀ-ÿ]\.-?){2,}$")
+
+
+def heading_rejection(heading):
+    """Why `heading` (the text after a heading number) is not a section
+    title, or None when it can be one. Pure, for tests."""
+    if _ATIP_STAMP.search(heading):
+        return "atip-stamp"
+    if _REPORTER.search(heading):
+        return "reporter-citation"
+    if _DATE_START.match(heading):
+        return "date"
+    if _LEADERS.search(heading):
+        return "toc-entry"
+    tail = heading.rstrip(" \"'”’»")
+    if tail.endswith(".") and not _INITIALISM_END.search(tail):
+        return "sentence"
+    if tail.endswith(","):
+        return "fragment"            # "15 Rue Fournier," (an address list)
+    return None
+
+
+# --- Headings wrapped at the PDF line end -----------------------------------
+# A long heading wraps: "13. Carrying out a decision of the" / "Immigration
+# Division". _HEADING sees only the first line, so (measured on the 2026-09
+# corpus) 73 EN and 107 FR section titles ended mid-phrase and the rest of the
+# title opened the chunk body. A title that ends where no title can end -- on
+# a function word, an open bracket or a dash -- takes the next line when that
+# line reads as its continuation: a short line opening on a letter, not
+# itself a numbered heading or an appendix title, and the joined title still
+# passes heading_rejection (so a body sentence ending in '.' is never pulled
+# in). Only the title changes: the section number, and with it the chunk id,
+# and the chunk text stay exactly as printed.
+_OPEN_END = {
+    "en": frozenset("of the a an to in for with by and or under from at"
+                    .split()),
+    "fr": frozenset("de la le les du des à aux dans que en et pour par sur au"
+                    .split()),
+}
+_OPEN_TAIL = ("(", "[", "–", "—", "-")
+_CONTINUATION_MAX = 80    # a wrapped title's second line is a short line
+_CONTINUATION_LINES = 2   # at most this many lines are joined on
+
+
+def _open_ended(heading, lang):
+    words = heading.split()
+    return bool(words) and (words[-1].lower() in _OPEN_END[lang]
+                            or heading.endswith(_OPEN_TAIL))
+
+
+def complete_heading(heading, following, lang="en"):
+    """`heading` with the line(s) it wrapped onto joined back on.
+
+    `following` is the normalised, non-empty lines after the heading line
+    and before the next heading. A hyphen glued to its word ("non-",
+    "personnes-") closes up with the next line, keeping the hyphen; any
+    other join takes a space. Pure, for tests."""
+    for line in following[:_CONTINUATION_LINES]:
+        if not _open_ended(heading, lang):
+            break
+        if (len(line) > _CONTINUATION_MAX or not line[0].isalpha()
+                or _HEADING.match(line) or _APPENDIX.match(line)):
+            break
+        glued = heading.endswith("-") and not heading.endswith(" -")
+        joined = heading + ("" if glued else " ") + line
+        if heading_rejection(joined):
+            break
+        heading = joined
+    return heading
+
+
+# A word the typesetting split with a space ("Impos er, modifier ou annuler",
+# "resolut ion"). The halves close up when the whole word is at least this
+# many times as common in the chapter as the rarer half -- the rule
+# caselaw._join_lines uses for the Federal Courts Reports ("a part" stays two
+# words).
+_WORD = re.compile(r"[^\W\d_]+")
+_JOIN_RATIO = 3
+
+
+def chapter_words(lines):
+    """Lower-cased word counts over a chapter's lines (for
+    repair_split_words and the line-break hyphens, _common.dehyphenate)."""
+    return document_words(lines)
+
+
+def repair_split_words(heading, words):
+    """`heading` with space-split words closed up (see _JOIN_RATIO). The
+    first half must be a bare word; the second must open lower-case, and
+    may carry trailing punctuation ("er,"). Pure, for tests."""
+    out = []
+    for tok in heading.split(" "):
+        prev = out[-1] if out else ""
+        tail = _WORD.match(tok)
+        if (prev and _WORD.fullmatch(prev) and tail and tok[0].islower()):
+            b = tail.group(0)
+            whole = words.get((prev + b).lower(), 0)
+            rarer = min(words.get(prev.lower(), 0), words.get(b.lower(), 0))
+            if whole and whole >= _JOIN_RATIO * rarer:
+                out[-1] = prev + tok
+                continue
+        out.append(tok)
+    return " ".join(out)
+
+
+# How far a chapter's major number may jump between two accepted headings.
+# Extraction can lose a heading or two; a bigger jump is a list item, a
+# date or a page number ("27 March 1992" after s. 14).
+_MAX_JUMP = 3
+
+
+def _num_key(num):
+    major, _, minor = num.partition(".")
+    return int(major), int(minor or 0)
+
+
+def _step_ok(prev, nxt):
+    (p_major, p_minor), (n_major, n_minor) = prev, nxt
+    if n_major == p_major:
+        return n_minor > p_minor
+    return p_major < n_major <= p_major + _MAX_JUMP and n_minor <= 2
+
+
+def numbering_run(nums, substantive=None):
+    """Indices of the heading numbers that form the chapter's numbering.
+
+    The best-scoring subsequence, in document order, in which each number
+    continues the previous one (same major with a larger minor, or the next
+    major -- up to _MAX_JUMP ahead -- at its start), beginning near 1.
+    Taking the best run rather than checking greedily means one stray
+    "5 Something" early on cannot lock out the real 4, 4.1, ... that follow.
+
+    `substantive[k]` says whether candidate k heads real text (more than a
+    line or two before the next candidate). A heading that does outweighs any
+    number of ones that do not, so a table of contents printed without dot
+    leaders -- ENF 14's OCR lists "1 What this chapter is about", "2 Program
+    objectives", ... as plain lines -- can never win over the body headings
+    it duplicates; among equals, more headings win. Ties keep the earliest
+    occurrence, so a heading repeated at a page break stays where it first
+    appears and the repeat reads as body text. A numbered annex that restarts
+    at 1 (ENF 19's IRB-CBSA MOU) is not a continuation either, so it stays
+    body text of the last real section instead of posing as the chapter's own
+    ss. 1-13. Pure, for tests."""
+    keys = [_num_key(n) for n in nums]
+    if substantive is None:
+        substantive = [True] * len(keys)
+    weight = [1000 if s else 1 for s in substantive]
+    best, back = [0] * len(keys), [-1] * len(keys)
+    for j, key in enumerate(keys):
+        if key[0] <= _MAX_JUMP and key[1] <= 2:
+            best[j] = weight[j]
+        for i in range(j):
+            if (best[i] and best[i] + weight[j] > best[j]
+                    and _step_ok(keys[i], key)):
+                best[j], back[j] = best[i] + weight[j], i
+    if not any(best):
+        return set()
+    j = max(range(len(keys)), key=lambda x: (best[x], -x))
+    keep = set()
+    while j != -1:
+        keep.add(j)
+        j = back[j]
+    return keep
+
+
+# --- ATIP release furniture -------------------------------------------------
+# The two UNOFFICIAL chapters are access-to-information releases: every page
+# of ENF 2 carries an OCR'd IRCC wordmark, the "Information disclosed under
+# the Access to Information Act" banner in both languages, the printed page
+# number and the release stamp "1 A-2023-83511-000094"; ENF 14's OCR carries
+# the intranet print furniture. None of it is guidance, and it used to sit in
+# the chunk text (and, for the stamp, in the section title). It is stripped;
+# the stamp is kept only as a page pinpoint in the citation. ATIP exemption
+# markers ("s.16(1)(b)") are kept: they say text was withheld there.
+_STAMP_IN_LINE = re.compile(r"(?:\b\d{1,3}\s*)?(?<![A-Za-z])A-\d{4}-\d+-\d+\b")
+_ATIP_FURNITURE = [re.compile(p) for p in (
+    r"^I\s*[•·]\s*I$",
+    r"^lmm\S*",                          # OCR of the 'Immigration' wordmark
+    r"^(?:and|et)\s+C[i1]t\S*\s+Canada\b",
+    r"^Information disclosed under the Access to Information Act$",
+    r"^L[’'`]information d\S* en vertu de la loi sur\b",
+    r"^DISTRIBUTION LIMITED TO CIC AND THE CBSA$",
+    r"^Date Last Modified: \d{4}-\d{2}-\d{2}$",
+    r"^Previous\s+[I|l1]\s+Next$",
+    r"^Page\s+\S{1,3}(?:\s+Of\S*)?$",
+    r"^https?://cicintranet\.",
+    r"^UO/UO/LU 1$",
+    r"^ENF 14 1 OP 19 Criminal Rehabilitation$",
+)]
+_ATIP_BANNER = re.compile(
+    r"\s*Information disclosed under the Access to Information Act\s*")
+
+
+def clean_atip_pages(pages):
+    """[(line, stamp)] for an ATIP release, furniture removed.
+
+    `stamp` is the release's identifier for the page the line is printed on
+    (it sits at the page foot, so it labels the text above it), or None when
+    the page carries none (ENF 14's OCR has no stamps). Pure, for tests."""
+    rows = []
+    for page in pages:
+        kept, stamp = [], None
+        for line in page.split("\n"):
+            s = line.strip()
+            found = _ATIP_STAMP.search(s)
+            if found:
+                stamp = found.group(0)
+                s = _STAMP_IN_LINE.sub(" ", s).strip()
+                if not s:
+                    # The printed page number sits on the line above.
+                    while kept and not kept[-1].strip():
+                        kept.pop()
+                    if kept and re.fullmatch(r"\d{1,3}", kept[-1].strip()):
+                        kept.pop()
+                    continue
+                line = s
+            if any(p.search(s) for p in _ATIP_FURNITURE):
+                continue
+            if _ATIP_BANNER.search(line):
+                line = _ATIP_BANNER.sub(" ", line).strip()
+                if not line:
+                    continue
+            kept.append(line)
+        rows.extend((line, stamp) for line in kept)
+    return rows
+
+
+def _pinpoint(stamps):
+    stamps = [s for s in stamps if s]
+    if not stamps:
+        return ""
+    first, last = stamps[0], stamps[-1]
+    return first if first == last else f"{first}–{last.rsplit('-', 1)[1]}"
+
+
+# --- ENF 1's jurisprudence annex -------------------------------------------
+# ENF 1 closes with "Appendix A Some notable inadmissibility jurisprudence":
+# for each IRPA ground, a "Jurisprudence" list of case citations, each with
+# its holdings. It has no numbered headings of its own, so the heading regex
+# used to cut it at reporter volumes that happened to start a line ("55
+# D.L.R. (4th) 321 ..."). It is segmented by case instead: one segment per
+# case, headed by the style of cause, tagged with the ground it is listed
+# under. The French edition is laid out the same way ("Appendice A", "c.").
+_ANNEX_TITLE = {
+    1: re.compile(r"^(?:Appendix|Appendice)\s+A\s+(?:Some notable "
+                  r"inadmissibility jurisprudence|Décisions marquantes)"),
+}
+_ANNEX_LABEL = {
+    "en": "Appendix A — Some notable inadmissibility jurisprudence",
+    "fr": ("Appendice A — Décisions marquantes faisant partie de la "
+           "jurisprudence relative aux interdictions de territoire"),
+}
+_JURIS_LINE = re.compile(r"^Jurisprudence\s*:?$")
+# A citation line: a style of cause ("X v. Y" / "X c. Y", at most one
+# parenthetical before the "v.", or "X, Re") followed by the report: a year
+# in parentheses, a [year] volume, a bare volume and reporter, or a neutral
+# citation.
+_CASE = re.compile(
+    r"^(?P<name>[A-Z][^()\n]{0,60}?(?:\s\([^()\n]*\))?\s+(?:v|c)\.\s+[A-Z]"
+    r"[^\n]*?|[A-Z][\w'’. -]{1,40}, Re)"
+    r"(?=\s*\((?:\d{4}|[^()\n]*\d{4})\)|,?\s*\[\d{4}\]"
+    r"|,?\s+\d{1,3}\s+[A-Z]|,\s*\d{4}\s+(?:FCA|FC|SCC|CAF|CF|CSC)\b)")
+_GROUND = re.compile(r"^[AL](\d{1,3})(?:\((\d+(?:\.\d+)?)\))?")
+_PARTY_STOP = {"canada", "mci", "maci", "r", "re", "the", "minister",
+               "ministre", "attorney", "procureur", "solicitor",
+               "solliciteur", "secretary", "secretaire", "general", "her",
+               "majesty", "queen"}
+# A holding starts with a bullet glyph (the PDFs use a private-use-area
+# character); a citation-only entry has none.
+_HOLDING = re.compile(r"^[-•▪◦‣∙-]")
+
+
+def _ascii(text):
+    import unicodedata
+    return unicodedata.normalize("NFKD", text).encode("ascii",
+                                                      "ignore").decode()
+
+
+def _party_slug(name):
+    """The id-safe party name of a style of cause: the first party word that
+    is not the Crown, the Minister or 'Re' ("Canada (MCI) v. Fenner" ->
+    'fenner', "Baroud, Re" -> 'baroud', "D’Souza v. Canada" -> 'dsouza')."""
+    for side in re.split(r"\s(?:v|c)\.\s", name, maxsplit=1):
+        side = re.sub(r"\([^)]*\)", " ", side)
+        for tok in side.replace(",", " ").split():
+            slug = re.sub(r"[^a-z]", "", _ascii(tok).lower())
+            if slug and slug not in _PARTY_STOP:
+                return slug
+    return "case"
+
+
+def _ground_slug(ground):
+    m = _GROUND.match(ground or "")
+    if not m:
+        return ""
+    return m.group(1) + (f"-{m.group(2).replace('.', '-')}" if m.group(2)
+                         else "")
+
+
+def _annex_start(n, lines):
+    """Index of the annex's own title line (not its table-of-contents
+    entry), or None when the chapter has no jurisprudence annex."""
+    pat = _ANNEX_TITLE.get(n)
+    if not pat:
+        return None
+    hits = [i for i, line in enumerate(lines)
+            if pat.match(_norm(line)) and not _LEADERS.search(line)]
+    return hits[-1] if hits else None
+
+
+def _is_holding(raw, text):
+    """A holding line: bullet glyph first (English edition), or indented
+    prose (the French edition's bullets do not survive extraction, but the
+    indent does). Citation continuations start flush, or with a digit or a
+    parenthesis."""
+    return bool(_HOLDING.match(text) or re.match(r"^\s+[^\W\d_]", raw))
+
+
+def annex_segments(rows):
+    """Split annex rows [(line, pin)] into per-case segments.
+
+    Returns dicts {ground, case, absorbed, rows}: the ground is the provision
+    line the case is listed under ("A36(1)(a), (b) and (c)"), the case its
+    style of cause, rows its normalised (line, pin) pairs. The leading text
+    before the first case is one segment with case None. A citation-only
+    entry (no holding under it) is folded into the case before it under the
+    same ground rather than left as a stub -- so a ground listed as bare
+    citations (ENF 1's A38 list) becomes one segment, `absorbed` counting the
+    cases folded in. Never across grounds. Pure, for tests."""
+    items = [(_norm(raw), pin, raw) for raw, pin in rows]
+    items = [it for it in items if it[0]]
+    segs = [{"ground": "", "case": None, "rows": [], "holding": False}]
+    ground, pending = "", []
+    for k, (text, pin, raw) in enumerate(items):
+        nxt = items[k + 1][0] if k + 1 < len(items) else ""
+        if _JURIS_LINE.match(nxt) and not _JURIS_LINE.match(text):
+            ground = text                      # the provision heading
+            pending = [(text, pin)]
+            continue
+        if _JURIS_LINE.match(text) and pending:
+            pending.append((text, pin))
+            continue
+        prev = segs[-1]["rows"][-1][0] if segs[-1]["rows"] else ""
+        continuation = prev.count("(") > prev.count(")")
+        m = None if continuation else _CASE.match(text)
+        if m:
+            segs.append({"ground": ground, "case": _norm(m.group("name")),
+                         "rows": pending + [(text, pin)], "holding": False})
+        else:
+            segs[-1]["rows"].extend(pending + [(text, pin)])
+            segs[-1]["holding"] |= _is_holding(raw, text)
+        pending = []
+    merged = []
+    for seg in segs:
+        prev = merged[-1] if merged else None
+        if (seg["case"] and not seg["holding"] and prev and prev["case"]
+                and prev["ground"] == seg["ground"]):
+            prev["rows"].extend(seg["rows"])
+            prev["absorbed"] += 1
+            continue
+        merged.append({"ground": seg["ground"], "case": seg["case"],
+                       "absorbed": 0, "rows": seg["rows"]})
+    # A citation-only entry that opens its ground has nothing before it to
+    # join ("Bellido v. MCI, [2005] FC 452" heads the A40 list with no
+    # holding). Too short to stand alone, it would be dropped; it is folded
+    # into the case after it instead, which carries the same ground line.
+    out = []
+    for k, seg in enumerate(merged):
+        short = len("\n".join(t for t, _ in seg["rows"])) <= 120
+        nxt = merged[k + 1] if k + 1 < len(merged) else None
+        if (seg["case"] and short and nxt and nxt["case"]
+                and nxt["ground"] == seg["ground"]):
+            nxt["rows"][:0] = [r for r in seg["rows"]
+                               if r not in nxt["rows"][:2]]
+            nxt["absorbed"] += 1 + seg["absorbed"]
+            continue
+        out.append(seg)
+    return out
+
+
+_FRONT_LABEL = {"en": "Cover and updates to chapter",
+                "fr": "Page couverture et mises à jour du chapitre"}
+# A contents entry that ends in a short dot run and its page number
+# ("11. Rules applicable to appeals ... 23"): too few dots for _LEADERS.
+_TOC_PAGE = re.compile(r"\.{2,}\s*\d{1,3}$")
+
+
+def _head_key(heading):
+    """A heading's comparable opening: case- and space-folded, trailing
+    dots and page number dropped, first 25 characters."""
+    h = re.sub(r"[\s.…]*\d{0,3}$", "", _norm(heading)).lower()
+    return re.sub(r"\s+", " ", h)[:25]
+
+
+def _body_heads(lines, start, end):
+    """{number: {heading key}} of every numbered-heading line in
+    lines[start:end] -- vetted or not, since a contents page lists what
+    the body prints."""
+    heads = {}
+    for line in lines[start:end]:
+        m = _HEADING.match(line.strip())
+        if m:
+            heads.setdefault(m.group(1), set()).add(_head_key(m.group(2)))
+    return heads
+
+
+def contents_entry(text, body_heads):
+    """True when a numbered front-matter line is a table-of-contents entry:
+    its number and opening words repeat a heading printed in the body, or it
+    ends in dots and a page number. Any other numbered line (an update note
+    that opens with the section it changed) is not. Pure, for tests."""
+    m = _HEADING.match(text)
+    if not m:
+        return False
+    if _TOC_PAGE.search(text):
+        return True
+    key = _head_key(m.group(2))
+    for body in body_heads.get(m.group(1), ()):
+        k = min(len(key), len(body))
+        if k >= 8 and key[:k] == body[:k]:
+            return True
+    return False
+
+# --- Trailing appendices ------------------------------------------------------
+# Most chapters end in lettered appendices (sample letters, forms, MOUs, case
+# lists). Many number their own items from 1, which numbering_run rightly
+# refuses as chapter sections -- so without this their text would be cited as
+# the chapter's last section (ENF 24's Article 1F case law as "s. 9.14").
+# An appendix title line counts only after the last numbered section (so the
+# cross-references and update notes that mention appendices mid-chapter are
+# ignored), and the titles must run A, B, C ... through numbering_run's own
+# logic, so a list of the appendices printed ahead of them (ENF 8) loses to
+# the appendices themselves.
+_APPENDIX = re.compile(r"^(?:Appendix|Annex|Appendice|Annexe)\s+([A-Z]|\d{1,2})"
+                       r"\b\s*[:–—-]?\s*(.*)$")
+
+
+def appendix_marks(lines, start, end):
+    """[(line index, letter, title)] of the appendix title lines in
+    lines[start:end]. Pure, for tests."""
+    cands = []
+    for i in range(start, end):
+        s = _norm(lines[i])
+        m = _APPENDIX.match(s)
+        if not m or len(s) > 120 or _LEADERS.search(s):
+            continue
+        title = m.group(2).strip()
+        # A cross-reference fragment ("Appendix C.)", "Annexe 1).") has no
+        # title after the letter. A real title may end in "(c)" or wrap
+        # mid-name ("... decision in Baker v."), so endings are not tested.
+        if not title or title[0] in ".),;":
+            continue
+        cands.append((i, m.group(1), title))
+    if not cands:
+        return []
+    nums = [str(ord(letter) - 64) if letter.isalpha() else letter
+            for _i, letter, _t in cands]
+    spans = [len(_norm(" ".join(lines[i + 1:(cands[k + 1][0]
+                                             if k + 1 < len(cands) else end)])))
+             for k, (i, _l, _t) in enumerate(cands)]
+    keep = numbering_run(nums, [span > 120 for span in spans])
+    return [c for k, c in enumerate(cands) if k in keep]
+
+
+# --- chapter currency ---------------------------------------------------------
+_LISTING = re.compile(r"^(?:\d+(?:\.\d+)*\.?\s+)?(?:Listing by date|Liste par "
+                      r"date|Liste des mises à jour par date)\b", re.I)
+
+
+def chapter_updated(pages, today=None):
+    """The chapter's date: the latest ISO date on its first three pages
+    (the cover/header convention most chapters follow) or, failing that,
+    the latest date in its own 'Updates to chapter -- Listing by date'
+    block. ENF 2's ATIP copy prints no date on its cover; its listing,
+    on ATIP page A-2023-83511-000005, opens with 2021-08-25. Dates later
+    than `today` are ignored in the listing. Pure, for tests."""
+    dates = _DATE.findall("\n".join(pages[:3]))
+    if dates:
+        return max(dates)
+    from datetime import date
+    today = today or date.today().isoformat()
+    lines = "\n".join(pages[:10]).split("\n")
+    for i, line in enumerate(lines):
+        s = _norm(line)
+        if _LISTING.match(s) and not _LEADERS.search(s):
+            found = [d for d in _DATE.findall("\n".join(lines[i:i + 40]))
+                     if d <= today]
+            if found:
+                return max(found)
+    return ""
+
+
+def _pdf_text(path):
+    from pypdf import PdfReader
+    reader = PdfReader(str(path))
+    pages = []
+    for page in reader.pages:
+        try:
+            pages.append(page.extract_text() or "")
+        except Exception:
+            pages.append("")
+    return pages
+
+
+def _pieces(rows):
+    """Split normalised (line, pin) rows into chunk-sized pieces, keeping
+    each piece's pins: [(text, pinpoint)]. Same boundaries as before the
+    pins existed -- split_lines only ever cuts between lines, so walking its
+    pieces' line counts recovers each piece's rows."""
+    body = "\n".join(t for t, _ in rows)
+    texts = split_lines(body, 1800) if len(body) > 2400 else [body]
+    out, at = [], 0
+    for text in texts:
+        count = text.count("\n") + 1
+        out.append((text, _pinpoint([p for _, p in rows[at:at + count]])))
+        at += count
+    return out
+
+
+def _chunks_for(n, pages, url, lang="en", atip=False, stats=None):
+    """Chunks for one chapter. `atip` strips an ATIP release's furniture
+    and pinpoints its page stamps; `stats`, if given, collects the rejected
+    heading candidates by reason."""
+    title = TITLES.get(n, f"ENF {n}")
+    fr = lang == "fr"
+    # a line-final soft hyphen is the printed hyphen of a broken word; kept
+    # as '-' so it survives norm_ws for the dehyphenate pass below
+    pages = [mark_soft_breaks(page) for page in pages]
+    updated = chapter_updated(pages)
+    rows = (clean_atip_pages(pages) if atip
+            else [(line, None) for page in pages for line in page.split("\n")])
+    lines = [line for line, _ in rows]
+    annex_at = _annex_start(n, lines)
+    main_end = len(lines) if annex_at is None else annex_at
+    # Segment on the manual's own numbered headings when enough are found;
+    # PDF extraction is uneven, so fall back to plain size-splitting.
+    cands = []
+    for i, line in enumerate(lines[:main_end]):
+        m = _HEADING.match(line.strip())
+        if not m:
+            continue
+        heading = _norm(m.group(2))
+        why = heading_rejection(heading)
+        if why:
+            if stats is not None:
+                stats[why] = stats.get(why, 0) + 1
+            continue
+        cands.append((i, m.group(1), heading))
+    spans = [len(_norm(" ".join(lines[i + 1:(cands[k + 1][0] if k + 1 < len(cands)
+                                              else main_end)])))
+             for k, (i, _num, _h) in enumerate(cands)]
+    keep = numbering_run([num for _, num, _ in cands],
+                         [span > 120 for span in spans])
+    if stats is not None and len(keep) < len(cands):
+        stats["out-of-sequence"] = (stats.get("out-of-sequence", 0)
+                                    + len(cands) - len(keep))
+    marks = [c for k, c in enumerate(cands) if k in keep]
+    words = chapter_words(lines)
+
+    def full_title(heading, seg):
+        # seg[0] is the heading line itself; the title may wrap onto the
+        # lines after it (complete_heading).
+        return repair_split_words(
+            complete_heading(heading, [t for t, _ in seg[1:]], lang), words)
+
+    # (id key, section, marginal note, heading field, cite label, rows)
+    segments = []
+    if len(marks) >= 6:
+        # Front matter: the cover and the 'Updates to chapter' listing, which
+        # say when and how the chapter changed. Before the heading vetting it
+        # survived by accident, under a table-of-contents line taken for a
+        # heading; now it precedes the first heading, so it is kept as its
+        # own segment, minus the contents lines: dot-leader entries, and the
+        # numbered entries a contents page prints without leaders (ENF 14's
+        # OCR lists all 36 section titles that way) -- a block of every
+        # section title would compete with the sections themselves. A
+        # numbered line that is not a contents entry stays: the update
+        # listing's own notes often open with the section they changed
+        # ("7.5 The table for search levels previously found at 7.6 has
+        # been modified", ENF 12).
+        body_heads = _body_heads(lines, marks[0][0], main_end)
+        front = [(t, p) for t, p in _norm_rows(rows[:marks[0][0]])
+                 if not _LEADERS.search(t)
+                 and not contents_entry(t, body_heads)]
+        if len("\n".join(t for t, _ in front)) > 120:
+            heading = _FRONT_LABEL[lang]
+            label = f"{'liminaire' if fr else 'front matter'} ({heading})"
+            segments.append(("front", f"ENF {n} front matter", heading, "",
+                             label, "", front))
+        apps = appendix_marks(lines, marks[-1][0] + 1, main_end)
+        body_end = apps[0][0] if apps else main_end
+        for j, (i, num, heading) in enumerate(marks):
+            end = marks[j + 1][0] if j + 1 < len(marks) else body_end
+            seg = _norm_rows(rows[i:end])
+            if len("\n".join(t for t, _ in seg)) > 120:
+                heading = full_title(heading, seg)
+                label = (f"art. {num} ({heading})" if fr
+                         else f"s. {num} ({heading})")
+                segments.append((num.replace(".", "-"), f"ENF {n} s. {num}",
+                                 heading, "", label, "", seg))
+        carry = []
+        for j, (i, letter, heading) in enumerate(apps):
+            end = apps[j + 1][0] if j + 1 < len(apps) else main_end
+            seg = carry + _norm_rows(rows[i:end])
+            if len("\n".join(t for t, _ in seg)) <= 120 and j + 1 < len(apps):
+                # A title-only appendix (its form is an image: ENF 20's
+                # "Annex A - Detention Oversight Form") joins the next one
+                # rather than vanishing.
+                carry = seg
+                continue
+            carry = []
+            if len("\n".join(t for t, _ in seg)) > 120:
+                heading = full_title(heading, _norm_rows(rows[i:end]))
+                label = f"{'app.' if fr else 'App.'} {letter} ({heading})"
+                segments.append((f"app-{letter.lower()}",
+                                 f"ENF {n} App. {letter}", heading, "",
+                                 label, "", seg))
+    if not segments:
+        text = "\n".join(lines[:main_end])
+        at = 0
+        for k, piece in enumerate(split_lines(text, 1800), start=1):
+            count = piece.count("\n") + 1
+            seg = _norm_rows(rows[at:at + count])
+            at += count
+            if len("\n".join(t for t, _ in seg)) > 120:
+                heading = f"{title} (part {k})"
+                label = (f"art. {k} ({heading})" if fr
+                         else f"s. {k} ({heading})")
+                segments.append((str(k), f"ENF {n} s. {k}", heading, "",
+                                 label, "", seg))
+    if annex_at is not None:
+        segments.extend(_annex_as_segments(n, rows[annex_at:], lang))
+    out = []
+    for key, section, heading, ground, label, division, seg in segments:
+        pieces = _pieces(seg)
+        total = len(pieces)
+        for k, (piece, pin) in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            where = f", ATIP p. {pin}" if pin else ""
+            out.append({
+                "id": (f"enf-fr-{n}-{key}{suffix}" if fr
+                       else f"enf-{n}-{key}{suffix}"),
+                "doc_type": "memorandum",
+                "act_code": f"ENF-{n}",
+                "act_short": f"ENF {n}",
+                "act_name": (f"ENF {n} — {title} (édition française)" if fr
+                             else f"ENF {n} — {title}"),
+                "section": section,
+                "marginal_note": heading,
+                "part": title,
+                "division": division,
+                "heading": ground,
+                # line-break hyphens closed up inside the chunk only, after
+                # the titles, ids and pins are fixed (2026-09-22)
+                "text": dehyphenate(piece, words),
+                "history": "",
+                "last_amended": updated,
+                "current_to": updated,
+                "citation": f"ENF {n}, {label}{note}{where}",
+                "source_url": url,
+                **({"lang": "fr"} if fr else {}),
+            })
+    return out
+
+
+def _annex_as_segments(n, rows, lang):
+    """ENF 1's jurisprudence annex as per-case segments (annex_segments)."""
+    fr = lang == "fr"
+    app = "app. A" if fr else "App. A"
+    segments = []
+    for seg in annex_segments(rows):
+        if seg["case"] is None:
+            heading, key, label = (_ANNEX_LABEL[lang].split(" — ", 1)[1],
+                                   "app-a", app)
+            ground = ""
+        else:
+            ground = seg["ground"]
+            gslug = _ground_slug(ground)
+            key = "-".join(p for p in ("app-a", gslug,
+                                       _party_slug(seg["case"])) if p)
+            heading = seg["case"]
+            k = seg["absorbed"]
+            if k:
+                heading += (f" (et {k} autre{'s' if k > 1 else ''} "
+                            f"décision{'s' if k > 1 else ''})" if fr else
+                            f" (and {k} more case{'s' if k > 1 else ''})")
+            sep = " : " if fr else ": "
+            label = (f"{app} ({ground}{sep}{heading})" if ground
+                     else f"{app} ({heading})")
+        if len("\n".join(t for t, _ in seg["rows"])) > 120:
+            segments.append((key, f"ENF {n} App. A", heading, ground, label,
+                             _ANNEX_LABEL[lang], seg["rows"]))
+    return segments
+
+
+def _norm_rows(rows):
+    """Normalised non-empty (line, pin) rows."""
+    return [(t, p) for t, p in ((_norm(line), p) for line, p in rows) if t]
+
+
+def _norm_block(lines):
+    return "\n".join(p for p in (_norm(l) for l in lines) if p)
+
+
+_HTTP_404 = re.compile(r"\b404\b")
+
+
+def is_missing_upstream(exc):
+    """True when a fetch failed because the chapter simply is not there (404).
+
+    _CHAPTERS is a blind range(1, 41) probe, so on a healthy run most numbers
+    404 -- and a chapter IRCC RETIRES starts 404ing too. That is this module's
+    documented signal to drop a chapter, so a 404 must not be preserved
+    through: doing so would pin every withdrawn chapter in the corpus forever.
+    Every other fetch error (TLS, timeout, a dropped connection) is a real
+    failure the stored copy should survive. The two fetch paths report
+    differently -- urllib raises HTTPError with a status, the PowerShell
+    fallback raises CalledProcessError with the status only in its stderr
+    prose ("The remote server returned an error: (404) Not Found."), so that
+    side is a text match.
+    """
+    if isinstance(exc, urllib.error.HTTPError):
+        return exc.code == 404
+    stderr = getattr(exc, "stderr", "") or ""
+    if isinstance(stderr, bytes):
+        stderr = stderr.decode("utf-8", "replace")
+    # the fetch script's own status line decides when it is there (since the
+    # verification fixes of 2026-09-22): a 5xx error page that happens to
+    # contain '404' is not a retirement; without the marker, the prose match
+    status = powershell_status(stderr)
+    if status is not None:
+        return status == 404
+    return bool(_HTTP_404.search(stderr))
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of chapters this run could not produce.
+
+    A chapter that stops fetching for any reason OTHER than a 404 -- a dropped
+    network, a TLS refusal, a PDF that suddenly has no text layer -- has not
+    been retired, it just did not arrive, and losing guidance that is still in
+    force is the worse error. So the stored copy stays and the run says so
+    loudly enough to be investigated. Chapters that 404 (retired, or never
+    existed) never enter `failures` and are not resurrected.
+    Returns (chunks, preserved act_codes).
+    """
+    failed = {f"ENF-{n}" for n, _why in failures}
+    kept, preserved = preserve_dropped(chunks, stored, key=lambda c: c["act_code"],
+                                       only=failed)
+    # A kept chapter goes back in its place, not at the end: on 2026-10-20 the
+    # kept ENF 2 and ENF 14 followed ENF 34 in enf.json. The sort is stable, so
+    # each chapter's own order is unchanged.
+    return sorted(kept, key=_chapter_number), preserved
+
+
+def _chapter_number(chunk):
+    """ENF chapter number of a chunk's act_code ('ENF-14' -> 14); anything else last."""
+    m = re.fullmatch(r"ENF-(\d+)", chunk.get("act_code", ""))
+    return int(m.group(1)) if m else float("inf")
+
+
+def build(chapters=_CHAPTERS, allow_shrink=False, force=False, lang="en"):
+    # Returns the chunks written, or False if the write was refused.
+    fr = lang == "fr"
+    all_chunks, skipped, failures = [], [], []
+    for n in chapters:
+        unofficial = None if fr else UNOFFICIAL.get(n)
+        if fr and n in _FR_STALE:
+            skipped.append(n)
+            continue
+        if fr:
+            url = _URL_OVERRIDES_FR.get(n, BASE_FR.format(n=n))
+        else:
+            url = (unofficial["url"] if unofficial
+                   else _URL_OVERRIDES.get(n, BASE.format(n=n)))
+        dest = RAW / (f"enf{n:02d}-fra.pdf" if fr else f"enf{n:02d}.pdf")
+        try:
+            # The ATIP host is an ordinary WordPress site; canada.ca needs
+            # the PowerShell TLS workaround.
+            fetch_cached(url, dest, powershell="canada.ca" in url, pause=1.0,
+                         force=force)
+        except Exception as exc:
+            skipped.append(n)
+            if not is_missing_upstream(exc):
+                # Not a 404: the chapter may well still be published, so it
+                # counts as a failure and its stored chunks are preserved.
+                failures.append((n, f"fetch failed: {type(exc).__name__}: {exc}"))
+            if dest.exists():
+                dest.unlink()      # don't cache an error page as a PDF
+            continue
+        try:
+            pages = _pdf_text(dest)
+            if sum(len(p) for p in pages) < 500:
+                # Image-scan PDF: fall back to the OCR cache (form-feed
+                # separated pages), produced offline via Windows OCR.
+                ocr = RAW / (f"enf{n:02d}-fra-ocr.txt" if fr
+                             else f"enf{n:02d}-ocr.txt")
+                if ocr.exists():
+                    pages = ocr.read_text(encoding="utf-8").split("\f")
+                    print(f"  ENF {n:2d}: using OCR cache "
+                          f"({sum(len(p) for p in pages)} chars)")
+                else:
+                    print(f"  !! ENF {n}: no text layer and no OCR cache "
+                          f"({ocr.name}); skipping")
+                    failures.append((n, f"no text layer and no {ocr.name}"))
+                    continue
+            rejected = {}
+            chunks = _chunks_for(n, pages, url, lang=lang,
+                                 atip=bool(unofficial), stats=rejected)
+            if rejected:
+                print(f"  ENF {n:2d}: heading candidates rejected: "
+                      + ", ".join(f"{k} {v}" for k, v in
+                                  sorted(rejected.items())))
+            if not chunks:
+                # Fetched fine, parsed to nothing: the PDF's heading structure
+                # changed under us. That is the shape the D-memo loss took, so
+                # treat it as a failure and keep the stored chapter.
+                print(f"  !! ENF {n}: no chunks parsed; keeping stored copy")
+                failures.append((n, "no content parsed"))
+                continue
+        except Exception as exc:
+            print(f"  !! ENF {n}: parse failed: {type(exc).__name__}: {exc}")
+            failures.append((n, f"parse failed: {type(exc).__name__}: {exc}"))
+            continue
+        if unofficial:
+            for c in chunks:
+                c["act_name"] = f"ENF {n} — {unofficial['title']}"
+                c["text"] = _UNOFFICIAL_BANNER + c["text"]
+                c["citation"] += " (unofficial ATIP copy)"
+                c["history"] = unofficial["note"]
+                c["source_status"] = ATIP_STATUS
+        print(f"  ENF {n:2d}: {len(chunks)} chunks"
+              + (" [UNOFFICIAL ATIP]" if unofficial else ""))
+        all_chunks.extend(chunks)
+    uniquify_ids(all_chunks)
+    out = PROCESSED_DIR / ("enf-fr.json" if fr else "enf.json")
+    # Preserved chunks keep the ids they were stored with, so they are added
+    # after uniquify_ids -- ids embed the chapter number, and a preserved
+    # chapter is by definition absent from this run's chunks.
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(out))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} chapter(s) "
+              f"that failed this run: {', '.join(preserved)}")
+    if not write_corpus(out, all_chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"enf: {len(all_chunks)} chunks -> {out.name} "
+          f"(chapters absent upstream: {skipped})")
+    return all_chunks
+
+
+def main():
+    chunks = build(allow_shrink="--allow-shrink" in sys.argv,
+                   force="--force" in sys.argv,
+                   lang="fr" if "--fr" in sys.argv else "en")
+    sys.exit(1 if chunks is False else 0)
+
+
+if __name__ == "__main__":
+    main()

canlex/eval.py

@@ -0,0 +1,310 @@
+"""Measure CanLex retrieval quality against a curated question set.
+
+Each item in data/eval/questions.json pairs a realistic legal question with the
+provision(s) or case(s) that answer it. This runs every question through the
+retrieval index and reports Hit@k and MRR. Re-run it after any retrieval change
+-- a new reranker, different embeddings, a chunking tweak -- to see whether
+quality moved, and read the "Misses" list to see exactly what to fix.
+
+Question schema (per item):
+    query      the question text
+    answers    [[act_short, section], ...]; empty section matches any chunk
+    style      optional tag ('standard' when absent): 'fact-pattern',
+               'keyword', 'typo', 'french', 'no-answer', 'criminality',
+               'lookup' (a section lookup in the officer's phrasing, the
+               pinned provision as gold -- 'IRPA 36(1)(a)', 'ss. 17 and 18
+               of IRPR'; 2026-09-21), ... -- metrics are reported per style
+               so a hard new slice cannot silently drag the comparable
+               legacy numbers
+    added      optional YYYY-MM-DD: the date the item joined the set. Any
+               dated item is outside the frozen legacy slice.
+    holdout    optional bool; holdout questions are excluded from parameter
+               sweeps (sweep.py) so knob choices cannot fit their noise --
+               eval.py reports them separately
+    must_see   optional list of keys that should ALL be in view (the top
+               VIEW_K results): {act_short, marginal_note}, {act_short,
+               section}, or {act_short} alone for a case (any paragraph).
+               Guidance is keyed by marginal note, so a re-chunk that
+               renumbers its sections does not invalidate the key. A
+               must_see key is not an answer: it never scores Hit@k.
+    'no-answer' style items carry no answers: they exist to calibrate the
+    confidence signal (a good corpus-coverage score should be LOW on them),
+    reported as a separate table, never as misses.
+
+Beyond Hit@k and MRR, each slice reports (roadmap T5.1):
+    cov@6    mean share of an item's must_see keys in the top VIEW_K results
+             (items with must_see only)
+    guid@6   share of items with a guidance result (doc_type 'memorandum':
+             ENF, PDI, IRB papers, IRCC guides) in the top VIEW_K that is an
+             answer or a must_see key (items naming guidance only)
+    Each is printed twice, 'a/b': a over the ranked view, b with the
+    official guidance the index co-surfaces for that view ('read next',
+    LegislationIndex._cosurface_guidance), which follows the results and
+    is never ranked.
+
+    py -m canlex.eval
+"""
+import json
+import os
+import sys
+from collections import defaultdict
+
+from .config import ROOT
+from .index import LegislationIndex
+
+QUESTIONS = ROOT / "data" / "eval" / "questions.json"
+EVAL_TOP_K = 20      # search depth, so ranks past the usual 6 are still visible
+VIEW_K = 6           # what the officer sees: cov@6 and guid@6 read this window
+GUIDANCE_DOC_TYPES = frozenset({"memorandum"})
+
+# The historical baseline slice: styles that existed before the 2026-07
+# expansion. Numbers over this slice stay comparable across commits -- which
+# only holds if the slice is frozen, so a question added later carries an
+# `added` date (YYYY-MM-DD) and is excluded here while still reported in its
+# style slice. Without this, eleven hard questions added on 2026-09-09
+# moved legacy Hit@1 by 0.05 with no existing question regressing.
+_LEGACY_STYLES = {"standard"}
+
+
+def is_legacy(item):
+    return item["style"] in _LEGACY_STYLES and not item.get("added")
+
+
+def load_questions():
+    """The question set with defaults applied; None if the file is absent."""
+    if not QUESTIONS.exists():
+        return None
+    items = json.loads(QUESTIONS.read_text(encoding="utf-8"))
+    for item in items:
+        item.setdefault("style", "standard")
+        item.setdefault("holdout", False)
+    return items
+
+
+def sweep_questions():
+    """The slice parameter sweeps may see (sweep.py calls this): everything
+    except holdout items and the no-answer calibration set."""
+    items = load_questions() or []
+    return [i for i in items
+            if not i["holdout"] and i["style"] != "no-answer"]
+
+
+def _matches(result, answers):
+    """True if a search result is one of the gold answers (act + section).
+
+    A gold answer is [act, section]; an empty section matches any chunk of that
+    act/case (used for case-law answers, whose chunks carry no section number).
+    """
+    r_acts = {result.get("act_short", "").lower(),
+              result.get("act_code", "").lower()}
+    r_sec = result.get("section", "")
+    for act, section in answers:
+        if act.lower() in r_acts and (section == r_sec or section == ""):
+            return True
+    return False
+
+
+def _norm(text):
+    return " ".join((text or "").split()).lower()
+
+
+def must_match(result, key):
+    """True if a result satisfies one must_see key. The key's act_short may
+    be the act's short name or its code; a marginal_note compares case- and
+    whitespace-folded; an empty section, or no section and no marginal note,
+    matches any chunk of the act or case."""
+    if _norm(key.get("act_short")) not in {_norm(result.get("act_short")),
+                                           _norm(result.get("act_code"))}:
+        return False
+    if "marginal_note" in key:
+        return _norm(result.get("marginal_note")) == _norm(key["marginal_note"])
+    if "section" in key:
+        return key["section"] == "" or result.get("section", "") == key["section"]
+    return True
+
+
+def guidance_acts(chunks):
+    """Case-folded act_short and act_code of every guidance chunk, so an item
+    can tell whether its answers or must_see keys name guidance at all."""
+    out = set()
+    for c in chunks:
+        if c.get("doc_type") in GUIDANCE_DOC_TYPES:
+            out.add(_norm(c.get("act_short")))
+            out.add(_norm(c.get("act_code")))
+    out.discard("")
+    return frozenset(out)
+
+
+def score_item(item, results, read_next=(), guidance=frozenset()):
+    """Per-question measures from one search.
+
+    rank    1-based rank of the first result matching an answer (0 = absent).
+            Co-surfaced read-next results (marked 'cosurfaced') follow the
+            ranked list and are never ranked.
+    cov6    share of the item's must_see keys in the top VIEW_K (None
+            without must_see); cov6_rn adds `read_next`, the guidance the
+            index co-surfaces for that view.
+    g6      a guidance result in the top VIEW_K that is an answer or a
+            must_see key (None when the item names no guidance -- `guidance`
+            is guidance_acts()); g6_rn adds `read_next`."""
+    ranked = [r for r in results if not r.get("cosurfaced")]
+    answers = [tuple(a) for a in item.get("answers") or []]
+    rank = next((i for i, r in enumerate(ranked, start=1)
+                 if _matches(r, answers)), 0)
+    keys = item.get("must_see") or []
+    view = ranked[:VIEW_K]
+    view_rn = view + list(read_next)
+
+    def coverage(rows):
+        if not keys:
+            return None
+        return sum(1 for k in keys
+                   if any(must_match(r, k) for r in rows)) / len(keys)
+
+    wants = (any(_norm(a) in guidance for a, _ in answers)
+             or any(_norm(k.get("act_short")) in guidance for k in keys))
+
+    def guided(rows):
+        if not wants:
+            return None
+        return any(r.get("doc_type") in GUIDANCE_DOC_TYPES
+                   and (_matches(r, answers)
+                        or any(must_match(r, k) for k in keys))
+                   for r in rows)
+
+    return {"rank": rank, "cov6": coverage(view), "cov6_rn": coverage(view_rn),
+            "g6": guided(view), "g6_rn": guided(view_rn)}
+
+
+def _metrics(ranks):
+    n = len(ranks) or 1
+    hit = lambda k: sum(1 for r in ranks if 0 < r <= k) / n
+    mrr = sum(1.0 / r for r in ranks if r) / n
+    return hit(1), hit(3), hit(5), hit(10), mrr
+
+
+def summarize(scores):
+    """Slice aggregates over score_item() dicts."""
+    ranks = [s["rank"] for s in scores]
+    h1, h3, h5, h10, mrr = _metrics(ranks)
+    out = {"n": len(scores), "hit1": h1, "hit3": h3, "hit5": h5,
+           "hit10": h10, "mrr": mrr}
+    for field in ("cov6", "cov6_rn", "g6", "g6_rn"):
+        vals = [float(s[field]) for s in scores if s.get(field) is not None]
+        out[field] = sum(vals) / len(vals) if vals else None
+        out[f"n_{field}"] = len(vals)
+    return out
+
+
+def _report_line(label, scores):
+    m = summarize(scores)
+    line = (f"  {label:24} n={m['n']:<4} Hit@1 {m['hit1']:.2f}  "
+            f"Hit@3 {m['hit3']:.2f}  Hit@5 {m['hit5']:.2f}  "
+            f"Hit@10 {m['hit10']:.2f}  MRR {m['mrr']:.2f}")
+    if m["n_cov6"]:
+        line += f"  cov@6 {m['cov6']:.2f}/{m['cov6_rn']:.2f} (n={m['n_cov6']})"
+    if m["n_g6"]:
+        line += f"  guid@6 {m['g6']:.2f}/{m['g6_rn']:.2f} (n={m['n_g6']})"
+    print(line)
+
+
+def read_next_for(index, results, positions):
+    """The guidance chunks the index co-surfaces for the top VIEW_K of
+    `results` (what an officer asking for VIEW_K results would get)."""
+    top = [positions[r["id"]] for r in results
+           if not r.get("cosurfaced") and r.get("id") in positions][:VIEW_K]
+    return [index.chunks[i] for i in index._cosurface_guidance(top)[len(top):]]
+
+
+def run():
+    items = load_questions()
+    if items is None:
+        print(f"No question set at {QUESTIONS}.", file=sys.stderr)
+        return
+    # Sweep mode (CANLEX_EVAL_SWEEP=1, set by sweep.py): score only the
+    # sweep-visible slice and print the legacy vertical metric block its
+    # parser expects -- holdout questions stay blind to every knob choice.
+    sweep_mode = bool(os.environ.get("CANLEX_EVAL_SWEEP"))
+    if sweep_mode:
+        items = [i for i in items
+                 if not i["holdout"] and i["style"] != "no-answer"]
+    index = LegislationIndex()
+    positions = {c["id"]: i for i, c in enumerate(index.chunks)}
+    guidance = guidance_acts(index.chunks)
+    scored = []         # (item, score_item dict)
+    misses = []
+    no_answer = []      # (query, top confidence) for the calibration table
+    for item in items:
+        if item["style"] == "no-answer":
+            results = index.search(item["query"], top_k=3)
+            conf = results[0].get("confidence") if results else None
+            no_answer.append((item["query"], conf))
+            continue
+        results = index.search(item["query"], top_k=EVAL_TOP_K)
+        score = score_item(item, results,
+                           read_next_for(index, results, positions), guidance)
+        scored.append((item, score))
+        rank = score["rank"]
+        if rank == 0 or rank > 5:
+            ranked = [r for r in results if not r.get("cosurfaced")]
+            misses.append((item, [tuple(a) for a in item["answers"]], rank,
+                           ranked[0] if ranked else None))
+
+    all_scores = [s for _, s in scored]
+    if sweep_mode:
+        h1, h3, h5, h10, mrr = _metrics([s["rank"] for s in all_scores])
+        print(f"CanLex retrieval evaluation -- {len(all_scores)} questions "
+              f"(sweep slice)\n")
+        print(f"  Hit@1:   {h1:.2f}")
+        print(f"  Hit@3:   {h3:.2f}")
+        print(f"  Hit@5:   {h5:.2f}")
+        print(f"  Hit@10:  {h10:.2f}")
+        print(f"  MRR:     {mrr:.2f}")
+        return
+    by_style = defaultdict(list)
+    for it, s in scored:
+        by_style[it["style"]].append(s)
+
+    print(f"CanLex retrieval evaluation -- {len(all_scores)} scored questions")
+    source = getattr(index, "see_also_source", None)
+    print(f"  (cov@6 and guid@6 print as ranked-view/with-read-next; "
+          f"read-next source: {source or 'none -- off'})\n")
+    _report_line("ALL", all_scores)
+    _report_line("legacy (comparable)", [s for it, s in scored if is_legacy(it)])
+    for style in sorted(by_style):
+        if style not in _LEGACY_STYLES:
+            _report_line(f"style: {style}", by_style[style])
+    held = [s for it, s in scored if it["holdout"]]
+    if held:
+        _report_line("holdout (sweep-blind)", held)
+        # The release gate's holdout line (with legacy). A dated slice's
+        # holdouts join the line above -- the 2026-09-15 criminality slice's
+        # ten took it from Hit@1 0.75 to 0.66 with no comparable question
+        # moving -- so the gate reads the undated holdouts, frozen like
+        # legacy and comparable across commits.
+        _report_line("holdout (undated)",
+                     [s for it, s in scored if it["holdout"] and not it.get("added")])
+        _report_line("sweep-visible", [s for it, s in scored if not it["holdout"]])
+
+    if no_answer:
+        print("\nNo-answer calibration (confidence should be LOW here; the "
+              "weak-match hedge fires below 0.72):")
+        for query, conf in no_answer:
+            shown = f"{conf:.3f}" if conf is not None else "n/a"
+            flag = "" if (conf is None or conf < 0.72) else "  <-- OVER-CONFIDENT"
+            print(f"  {shown}  {query[:70]}{flag}")
+
+    if misses:
+        print(f"\n{len(misses)} miss(es) -- gold answer ranked >5 or absent:")
+        for item, answers, rank, top in misses:
+            gold = ", ".join(f"{a} s.{s}".rstrip(" s.") for a, s in answers)
+            where = f"ranked #{rank}" if rank else f"absent (searched {EVAL_TOP_K})"
+            got = (f"{top.get('act_short', '')} s.{top.get('section', '')}".rstrip(" s.")
+                   if top else "nothing")
+            print(f"  [{where}] ({item['style']}) {item['query']}")
+            print(f"      gold: {gold}   |   top result: {got}")
+    print()
+
+
+if __name__ == "__main__":
+    run()

canlex/ingest.py

@@ -0,0 +1,671 @@
+"""Ingest Justice Laws XML into structured, section-level JSON chunks."""
+import contextlib
+import re
+import sys
+import time
+import urllib.request
+import xml.etree.ElementTree as ET
+
+from ._common import (atomic_write_bytes, norm_ws as _norm, split_lines,
+                      with_retry, write_corpus)
+from .config import SOURCES, RAW_DIR, PROCESSED_DIR
+
+LIMS = "{http://justice.gc.ca/lims}"
+BLOCK_TAGS = {"Subsection", "Paragraph", "Subparagraph", "Clause", "Definition"}
+# A French consolidation heads its divisions 'SECTION 1', 'SECTION 7.1', 'SECTION IV'
+# where the English one says 'DIVISION'; the division field was empty on every
+# French chunk (2026-10-12: all 571 of the CASR 2012's). Upper case and a bare
+# label: an English heading 'Section 5 of the Act — Trafficking' (SOR/97-234) is
+# not a division.
+_FR_DIVISION = re.compile(r"SECTION [0-9IVXLC]+(?:\.\d+)?(?: - |$)")
+
+# A section whose entire rendered body is a placeholder marker -- "30 [Amendment]",
+# "162 [Amendments]", "5 [Related provision]" -- left where an enacting Act amended
+# another Act. Content-free; dropped at ingest. ([Repealed, ...] is deliberately
+# excluded here so repealed sections are kept and flagged instead.)
+#
+# The French consolidations print the same placeholders as "[Modification]",
+# "[Modifications]" and "[Disposition connexe]" (measured 2026-09-21: 229, 125
+# and 2 in the raw FR XML); until then the pattern was English-only and the
+# French twins kept 314 content-free stubs as live provisions. A stub may name
+# a RANGE of sections -- '204 et 205 [Modifications]', '211 à 214
+# [Modifications]', '30 to 32 [Amendments]' -- and 104 of those survived the
+# one-token rule (the audit follow-up of 2026-09-21).
+_AMENDMENT_STUB = re.compile(
+    r"^\S+(?:\s+(?:et|à|a|to|and|-|–)\s+\S+)?\s*\[(?:Amendments?|Related provisions?"
+    r"|Modifications?|Dispositions? connexes?)\]\s*$", re.IGNORECASE)
+
+# The semantic embedder (embed.py) only reads the first _MAX_BODY=2000 chars of
+# a chunk, so the tail of a long section is invisible to semantic search. A
+# section whose rendered text exceeds _SPLIT_THRESHOLD is therefore emitted as
+# several piece-chunks, each <= _PIECE_TARGET chars (under the embed window) and
+# split on subsection/line boundaries, so every part is embeddable. Sections at
+# or below the threshold are emitted unchanged -- same id, same embedding --
+# which keeps ~90% of legislation chunks (and their retrieval behaviour) intact.
+_SPLIT_THRESHOLD = 2400
+_PIECE_TARGET = 1800
+
+
+def _split_text(text, target=_PIECE_TARGET):
+    """Split rendered section text into <=target pieces at line boundaries,
+    folding small leading/trailing remnants into a neighbour so no piece is a
+    stub."""
+    return split_lines(text, target, fold_stubs=True)
+
+
+def _itertext(el):
+    return _norm("".join(el.itertext()))
+
+
+def _label_text(el):
+    """A <Label>'s text without its editorial footnote marker.
+
+    Justice Laws puts `<FootnoteRef idref="...">*</FootnoteRef>` inside the
+    Label of a provision whose in-force date is footnoted (IRPA ss. 189-193,
+    s. 188(4); the YCJA, the Cannabis Act, the CBSA Act...). _itertext kept
+    the '*', so 70 chunks were stored as section '*189', cited as 's. *189',
+    linked to a section-*189.html that does not exist, and an exact
+    get_section lookup found nothing. Only the Label reads use this: a
+    FootnoteRef inside a body <Text> stays in the text as published."""
+    parts = [el.text or ""]
+    for child in el:
+        if child.tag != "FootnoteRef":
+            parts.append("".join(child.itertext()))
+        parts.append(child.tail or "")
+    return _norm("".join(parts))
+
+
+# The first kilobyte of a Justice Laws consolidation names its root element;
+# a maintenance page or an error body does not, and must not be cached.
+_XML_ROOTS = (b"<Statute", b"<Regulation")
+
+
+def _xml_body(url):
+    """One attempt at reading a consolidation's bytes."""
+    req = urllib.request.Request(url, headers={"User-Agent": "CanLex/0.1"})
+    with urllib.request.urlopen(req, timeout=120) as resp:
+        return resp.read()
+
+
+def _download_xml(url, dest):
+    """Download a consolidation to `dest` atomically (a temporary file renamed
+    into place) and only when the body is Justice Laws XML, so a truncated or
+    non-XML response can never poison the cache for every later plain run.
+    A 5xx, a dropped connection or a timeout is retried once
+    (_common.with_retry); a 404 or a non-XML body is not."""
+    data = with_retry(lambda: _xml_body(url), what=url)
+    head = data[3:1024] if data[:3] == b"\xef\xbb\xbf" else data[:1024]
+    if not any(root in head for root in _XML_ROOTS):
+        raise ValueError(f"{url}: response is not Justice Laws XML "
+                         f"(no <Statute>/<Regulation> in the first KB); not cached")
+    atomic_write_bytes(dest, data)
+    time.sleep(1.0)  # be polite to the Justice Laws server between downloads
+    return dest
+
+
+def fetch_xml(code, force=False):
+    src = SOURCES[code]
+    RAW_DIR.mkdir(parents=True, exist_ok=True)
+    dest = RAW_DIR / f"{code}.xml"
+    if dest.exists() and not force:
+        return dest
+    print(f"  downloading {src['xml_url']}")
+    return _download_xml(src["xml_url"], dest)
+
+
+def _parse_or_evict(xml_path, code):
+    """parse_legislation, evicting the cached XML when it cannot be parsed:
+    a truncated body that passed the root check, or a cache written before
+    the check existed, would otherwise fail identically on every plain run
+    with nothing saying the cache is the cause."""
+    try:
+        return parse_legislation(xml_path, code)
+    except (ET.ParseError, ValueError) as exc:
+        with contextlib.suppress(OSError):
+            xml_path.unlink()
+        raise type(exc)(f"{exc} -- cached {xml_path.name} removed; re-run to "
+                        f"download it again") from exc
+
+
+def _heading_text(h):
+    """Render a <Heading> as 'LABEL - TITLE' when both parts are present."""
+    label, title = h.find("Label"), h.find("TitleText")
+    if label is not None or title is not None:
+        parts = [_label_text(label) if label is not None else "",
+                 _itertext(title) if title is not None else ""]
+        return " - ".join(p for p in parts if p)
+    return _itertext(h)
+
+
+def _render_block(el, is_section=False):
+    """Recursively render a provision element into readable, structured text."""
+    label = note = ""
+    inline, blocks = [], []
+    for child in el:
+        tag = child.tag
+        if tag == "Label":
+            label = _label_text(child)
+        elif tag == "MarginalNote":
+            if not is_section:  # a section's own marginal note is its title
+                note = _itertext(child)
+        elif tag == "Text":
+            inline.append(_itertext(child))
+        elif tag == "HistoricalNote":
+            continue
+        elif tag == "TableGroup":
+            # a table inside a provision (2026-10-12: CASR 2012 s. 78(2), IRPR
+            # s. 7.01, the Cannabis Regulations' s. 21): recursing into it kept
+            # only the lists nested in its cells and dropped every cell's text
+            blocks.append(_table_text(child))
+        elif tag in BLOCK_TAGS:
+            blocks.append(_render_block(child))
+        else:
+            nested = _render_block(child)
+            if nested:
+                blocks.append(nested)
+    head = " ".join(p for p in (label, f"[{note}]" if note else "", " ".join(inline)) if p)
+    if blocks:
+        return (head + "\n" if head else "") + "\n".join(b for b in blocks if b)
+    return head
+
+
+def _table_text(tablegroup):
+    """Render a CALS <TableGroup> as caption/title lines plus one
+    pipe-separated line per row. Column structure survives as ' | '
+    separators, which is enough for BM25/semantic matching and readable in a
+    result pane; no attempt is made to preserve spanned cells."""
+    out = []
+    for t in (tablegroup.find("Caption"), tablegroup.find("table/title")):
+        if t is not None:
+            txt = _itertext(t)
+            if txt:
+                out.append(txt)
+    for row in tablegroup.iter("row"):
+        cells = [_cell_text(e) for e in row.findall("entry")]
+        # an empty cell keeps its column: stripping it made the CASR 2012 Schedule 4
+        # row 'Section 14 | 5,000 |' (individual maximum only) read as a corporation
+        # amount. Only a row with no text at all is dropped.
+        if any(cells):
+            out.append(" | ".join(cells).strip())
+    return "\n".join(out)
+
+
+# Elements a table cell's words are separated at (CASR 2012 s. 78(2)'s conditions
+# are a <Provision><List><Item> tree; SOR/96-263's cells hold image captions and
+# footnotes). Any other element is inline (Emphasis, Language, a cross-reference)
+# and never splits a word.
+_CELL_BREAKS = BLOCK_TAGS | {"Provision", "List", "Item", "Label", "Text", "Footnote",
+                             "ImageGroup", "Caption", "AlternateText", "LineBreak"}
+
+
+def _cell_text(entry):
+    """One table cell on one line, every word of it kept: the cell's text with a
+    space at each structural boundary, so a nested list reads '(a) ... (b) ...'
+    instead of '(a)the officer'. A cell holding only inline text reads exactly as
+    its itertext."""
+    parts = []
+
+    def walk(el):
+        brk = " " if el.tag in _CELL_BREAKS else ""
+        parts.append(brk + (el.text or ""))
+        for child in el:
+            walk(child)
+            parts.append(child.tail or "")
+        parts.append(brk)
+
+    walk(entry)
+    return _norm("".join(parts))
+
+
+def _first_table_title(sched):
+    """The title of the table a schedule opens with, or '' -- the CASR 2012 names
+    its aerodrome schedules only there ('Class 1 Aerodromes')."""
+    for child in sched:
+        if child.tag in ("ScheduleFormHeading", "HistoricalNote"):
+            continue
+        title = child.find("table/title") if child.tag == "TableGroup" else None
+        return _itertext(title) if title is not None else ""
+    return ""
+
+
+def _sched_body(el):
+    """Render a <Schedule>'s content elements into readable text.
+
+    Observed shapes across the corpus: List/Item trees (CDSA I-IV, IRPR
+    1.1), DocumentInternal/Group/Provision prose (CDSA V-VI, IRPA's Refugee
+    Convention), and TableGroup grids (IRPR Schedule 2 violations, the AMPS
+    designated-provisions tables). Lists, groups and provisions reuse
+    _render_block, whose Label/Text recursion matches their structure;
+    tables get the dedicated renderer above so rows keep their shape."""
+    parts = []
+    for child in el:
+        tag = child.tag
+        if tag in ("ScheduleFormHeading", "HistoricalNote"):
+            continue
+        if tag == "TableGroup":
+            parts.append(_table_text(child))
+        elif tag == "GroupHeading":
+            parts.append(_heading_text(child))
+        elif tag in ("DocumentInternal", "Group", "RegulationPiece", "BillPiece"):
+            parts.append(_sched_body(child))
+        else:
+            rendered = _render_block(child)
+            if rendered:
+                parts.append(rendered)
+    return "\n".join(p for p in parts if p)
+
+
+# Schedule ids with dedicated handling: NifProvs is ingested separately as
+# not-in-force chunks; RelatedProvs (transitional provisions of past
+# amendments) is deliberately excluded -- see _nif_chunks.
+_SCHED_SKIP_IDS = {"NifProvs", "RelatedProvs"}
+
+
+def _heading_part(heading, tag):
+    """The full text of a ScheduleFormHeading child, or ''."""
+    el = heading.find(tag) if heading is not None else None
+    return _itertext(el) if el is not None else ""
+
+
+def _schedule_chunks(root, code, src, current_to):
+    """Chunks for an Act's substantive Schedules.
+
+    Previously every Schedule except NifProvs was silently dropped -- which
+    meant the CDSA's controlled-substance lists, the AMPS designated-
+    provisions tables and IRPA's Refugee Convention articles were absent
+    from the corpus even though the sections that depend on them were
+    ingested. A source can opt out with 'skip_schedules': True (the Customs
+    Tariff does: its schedule is the full tariff nomenclature, served
+    separately by tariff_schedule.py from the CBSA edition)."""
+    if src.get("skip_schedules"):
+        return []
+    chunks = []
+    for n, sched in enumerate(root.findall("Schedule"), start=1):
+        if sched.get("id") in _SCHED_SKIP_IDS:
+            continue
+        h = sched.find("ScheduleFormHeading")
+        label = _norm(h.findtext("Label") or "") if h is not None else ""
+        # the whole element's text: findtext stops at the first child, so
+        # '(Sections <XRefInternal>797</XRefInternal> and ...)' was stored as '(Sections'
+        title = _heading_part(h, "TitleText")
+        oref = _heading_part(h, "OriginatingRef")
+        label = label or (f"SCHEDULE ({n})" if not title else title)
+        table_title = "" if title else _first_table_title(sched)
+        repealed = sched.find("Repealed") is not None
+        body = "[Repealed]" if repealed else _sched_body(sched)
+        if not body:
+            continue
+        header = " — ".join(p for p in (title, oref) if p)
+        text = (header + "\n" if header else "") + body
+        base = {
+            "act_code": code,
+            "act_short": src["short"],
+            "act_name": src["name"],
+            "section": label.title(),
+            "marginal_note": title or table_title or label.title(),
+            "part": "SCHEDULES",
+            "division": "",
+            "heading": label.title(),
+            "history": "",
+            "last_amended": sched.get(f"{LIMS}lastAmendedDate", ""),
+            "in_force": sched.get(f"{LIMS}inforce-start-date", ""),
+            "status": "repealed" if repealed else "in force",
+            "current_to": current_to,
+            "citation": f"{src['short']}, {label.title()}",
+            "source_url": src["web_base"],
+        }
+        pieces = (_split_text(text)
+                  if len(text) > _SPLIT_THRESHOLD else [text])
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            if total == 1:
+                chunks.append({**base, "id": f"{code}-sched{n}", "text": piece})
+            else:
+                chunks.append({
+                    **base, "id": f"{code}-sched{n}-p{k}", "text": piece,
+                    "citation": f"{base['citation']} (part {k} of {total})"})
+    return chunks
+
+
+def _history(section):
+    note = section.find("HistoricalNote")
+    if note is None:
+        return ""
+    return "; ".join(_itertext(i) for i in note.iter("HistoricalNoteSubItem"))
+
+
+def parse_legislation(xml_path, code):
+    src = SOURCES[code]
+    data = xml_path.read_bytes()
+    if data[:3] == b"\xef\xbb\xbf":  # some Justice Laws XML files carry a UTF-8 BOM
+        data = data[3:]
+    root = ET.fromstring(data)
+    current_to = root.get(f"{LIMS}current-date", "")
+    body = root.find("Body")
+    if body is None:
+        # A schedule-only instrument (e.g. the Import Control List) has no
+        # <Body> at all -- its entire substance is in Schedules.
+        if not root.findall("Schedule"):
+            raise ValueError(f"{code}: no <Body> element (root <{root.tag}>)")
+        body = ()
+
+    headings, chunks = {}, []
+    for el in body:
+        if el.tag == "Heading":
+            try:
+                level = int(el.get("level", "0"))
+            except ValueError:
+                level = 0
+            headings = {k: v for k, v in headings.items() if k < level}
+            headings[level] = _heading_text(el)
+            continue
+        if el.tag != "Section":
+            continue
+
+        label_el = el.find("Label")
+        number = _label_text(label_el) if label_el is not None else ""
+        note_el = el.find("MarginalNote")
+        marginal = _itertext(note_el) if note_el is not None else ""
+        body_text = _render_block(el, is_section=True)
+
+        # Drop content-free amendment/related-provision stubs. When an Act
+        # amends OTHER Acts, the consolidation keeps a placeholder section whose
+        # whole body is just a marker -- e.g. "30 [Amendment]" -- the actual
+        # text living in the Act being amended. These carry no substance, yet
+        # were being indexed and rendered with an 'in force' currency line as if
+        # they were live provisions (117 in the CBSA Act alone). Skip them.
+        # (Repealed sections, "<n> [Repealed, ...]", are NOT dropped -- they are
+        # flagged status='repealed' and kept so a section lookup still informs.)
+        if _AMENDMENT_STUB.match(body_text):
+            continue
+
+        if number:
+            chunk_id = f"{code}-s{number}"
+            citation = f"{src['short']}, s. {number}"
+            source_url = f"{src['web_base']}section-{number}.html"
+        else:
+            # An unnumbered <Section> is a preamble or enacting recital. Keep it
+            # unless it has no text; the LIMS id keeps the chunk id unique when
+            # an Act contains more than one such block.
+            if not body_text:
+                continue
+            number = marginal or "Preamble"
+            marginal = marginal or "Preamble"
+            lims_id = el.get(f"{LIMS}id") or str(len(chunks))
+            chunk_id = f"{code}-pre-{lims_id}"
+            citation = f"{src['short']}, {number}"
+            source_url = src["web_base"]
+
+        part = next((v for _, v in sorted(headings.items())
+                     if v.upper().startswith("PART")), "")
+        division = next((v for _, v in sorted(headings.items())
+                         if v.upper().startswith("DIVISION") or _FR_DIVISION.match(v)), "")
+        nearest = headings[max(headings)] if headings else ""
+
+        # Per-section coming-into-force date (lims:inforce-start-date is present
+        # on every section) and repeal status. A wholly-repealed section is a
+        # <Section> whose own <Text> is a <Repealed> marker (e.g. "[Repealed,
+        # 2018, c. 27, s. 174]"); a live section with a repealed subsection has
+        # the <Repealed> nested deeper, so we key on the direct Text/Repealed to
+        # flag only sections that are themselves no longer law.
+        in_force = el.get(f"{LIMS}inforce-start-date", "")
+        status = "repealed" if el.find("Text/Repealed") is not None else "in force"
+
+        base = {
+            "act_code": code,
+            "act_short": src["short"],
+            "act_name": src["name"],
+            "section": number,
+            "marginal_note": marginal,
+            "part": part,
+            "division": division,
+            "heading": nearest,
+            "history": _history(el),
+            "last_amended": el.get(f"{LIMS}lastAmendedDate", ""),
+            "in_force": in_force,
+            "status": status,
+            "current_to": current_to,
+            "source_url": source_url,
+        }
+
+        # Short sections (the large majority) stay one chunk with their
+        # original id and citation -- unchanged embedding and retrieval. Long
+        # ones are split into embeddable pieces; each piece keeps the section
+        # number (so citation lookups, section-ref pinning and the eval still
+        # match) and notes its part in the citation.
+        pieces = (_split_text(body_text)
+                  if len(body_text) > _SPLIT_THRESHOLD else [body_text])
+        n = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            if n == 1:
+                chunks.append({**base, "id": chunk_id, "text": piece,
+                               "citation": citation})
+            else:
+                chunks.append({**base, "id": f"{chunk_id}-p{k}", "text": piece,
+                               "citation": f"{citation} (part {k} of {n})"})
+
+    chunks.extend(_schedule_chunks(root, code, src, current_to))
+    chunks.extend(_nif_chunks(root, code, src, current_to))
+    return chunks
+
+
+def _nif_chunks(root, code, src, current_to):
+    """Chunks for the Act's AMENDMENTS NOT IN FORCE schedule.
+
+    Justice Laws consolidations carry enacted-but-not-yet-in-force amendments
+    in a Schedule with id='NifProvs' (each entry a RelatedOrNotInForce node:
+    a citation heading for the amending Act plus the amendment text as regular
+    Section elements). Previously the parser dropped these silently -- the
+    corpus's biggest currency blind spot, since a Royal-Assented amendment
+    days from changing the law was invisible until consolidation. They are
+    ingested with status='not-in-force': retrieval down-weights them below
+    live law and the server banners them, but a search for their subject
+    matter now surfaces the pending law instead of nothing.
+
+    The sibling Schedule id='RelatedProvs' (transitional related provisions)
+    is deliberately NOT ingested here: those are in-force companion rules of
+    past amendments, not pending law, and would need their own treatment.
+
+    Note the Label inside an entry is the AMENDING Act's section number, so
+    the chunk's 'section' is left empty -- these must never satisfy a lookup
+    or pin for the target Act's own section numbers.
+    """
+    nif = next((s for s in root.findall("Schedule")
+                if s.get("id") == "NifProvs"), None)
+    if nif is None:
+        return []
+    chunks = []
+    for n, entry in enumerate(nif.iter("RelatedOrNotInForce"), start=1):
+        title = entry.find("Heading/TitleText")
+        cite = _norm("".join(title.itertext())) if title is not None else ""
+        cite = cite.lstrip("—- ").strip()
+        sections = entry.findall("Section")
+        body = "\n".join(p for p in (_render_block(s, is_section=True)
+                                     for s in sections) if p)
+        if not body:
+            continue
+        note_el = sections[0].find("MarginalNote") if sections else None
+        marginal = _norm("".join(note_el.itertext())) if note_el is not None else ""
+        pieces = (_split_text(body)
+                  if len(body) > _SPLIT_THRESHOLD else [body])
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            part_note = "" if total == 1 else f" (part {k} of {total})"
+            chunks.append({
+                "id": f"{code}-nif-{n}{suffix}",
+                "act_code": code,
+                "act_short": src["short"],
+                "act_name": src["name"],
+                "section": "",
+                "marginal_note": marginal or f"Amendment not in force — {cite}",
+                "part": "AMENDMENTS NOT IN FORCE",
+                "division": "",
+                "heading": "Amendments not in force",
+                "text": piece,
+                "history": cite,
+                "last_amended": "",
+                "in_force": "",
+                "status": "not-in-force",
+                "current_to": current_to,
+                "citation": (f"{src['short']} — amendment not in force "
+                             f"({cite}){part_note}"),
+                "source_url": src["web_base"],
+            })
+    return chunks
+
+
+def ingest(code, force=False, allow_shrink=False):
+    print(f"Ingesting {code} ({SOURCES[code]['short']})...")
+    xml_path = fetch_xml(code, force=force)
+    chunks = _parse_or_evict(xml_path, code)
+    out = PROCESSED_DIR / f"{code}.json"
+    # An Act whose XML schema shifts parses to a fraction of its sections; the
+    # guard turns that into a refusal instead of a quietly gutted statute.
+    if not write_corpus(out, chunks, indent=2, allow_shrink=allow_shrink):
+        raise RuntimeError(f"{code}: refused to write a collapsed corpus")
+    print(f"  {len(chunks)} sections -> {out.name}")
+    return chunks
+
+
+# --- French corpus (phase 3 of bilingual support) ----------------------------
+# Justice Laws publishes every consolidation as parallel French XML in the
+# SAME schema (English element names, French text, identical section
+# numbering), so the whole parser -- schedules included -- works unchanged.
+# French chunks carry lang='fr', a '-fr-' id infix, French citation style
+# ('art.' / 'annexe') and the French web URL; retrieval filters by query
+# language (index.py), and get_section/xref/list_acts stay English-first.
+
+def _fr_xml_url(src):
+    return (src["xml_url"]
+            .replace("/eng/XML/", "/fra/XML/")
+            .replace("/XML/SOR-", "/XML/DORS-")  # règlements: SOR -> DORS
+            .replace("_c._", "_ch._"))        # C.R.C. chapters: c. -> ch.
+
+
+# The parser writes a few structural labels of its own, in English: the
+# SCHEDULES part, the AMENDMENTS NOT IN FORCE schedule and its entries, the
+# '(part k of n)' note on a split provision, the 'Preamble' and 'SCHEDULE (n)'
+# fallbacks. The French twins carried them verbatim (2026-09-21 audit:
+# 'SCHEDULES' on 845 French chunks, 'Amendments not in force' on 452, the
+# part note on every split article); these are the French consolidations' own
+# words. (field, English pattern, French replacement), applied in order.
+_FR_LABELS = (
+    ("part", re.compile(r"^SCHEDULES$"), "ANNEXES"),
+    ("part", re.compile(r"^AMENDMENTS NOT IN FORCE$"), "MODIFICATIONS NON EN VIGUEUR"),
+    ("heading", re.compile(r"^Amendments not in force$"), "Modifications non en vigueur"),
+    ("marginal_note", re.compile(r"^Amendment not in force — "), "Modification non en vigueur — "),
+    ("citation", re.compile(r" — amendment not in force \("), " — modification non en vigueur ("),
+    ("citation", re.compile(r"\(part (\d+) of (\d+)\)"), r"(partie \1 de \2)"),
+    ("citation", re.compile(r", Preamble$"), ", Préambule"),
+    ("section", re.compile(r"^Preamble$"), "Préambule"),
+    ("marginal_note", re.compile(r"^Preamble$"), "Préambule"),
+    ("section", re.compile(r"^Schedule \((\d+)\)$"), r"Annexe (\1)"),
+    ("heading", re.compile(r"^Schedule \((\d+)\)$"), r"Annexe (\1)"),
+    ("marginal_note", re.compile(r"^Schedule \((\d+)\)$"), r"Annexe (\1)"),
+)
+
+# The English section label a French one pairs with: a range reads '55 et
+# 56' / '211 à 214' in French and '55 and 56' / '211 to 214' in English, a
+# schedule 'Annexe 1' / 'Schedule 1', a form 'Formule 6' / 'Form 6' (322
+# French labels with no English twin of the same spelling in the 2026-09-21
+# corpus; this table pairs all but nine). Section labels are title-cased at
+# parse, so the connectors come in both cases.
+_SECTION_EN = (
+    (re.compile(r"^Annexes\b"), "Schedules"),
+    (re.compile(r"^Annexe\b"), "Schedule"),
+    (re.compile(r"^Formule\b"), "Form"),
+    (re.compile(r"^Préambule$"), "Preamble"),
+    (re.compile(r"\bDe La Partie\b"), "To Part"),
+    (re.compile(r" et "), " and "), (re.compile(r" Et "), " And "),
+    (re.compile(r" à "), " to "), (re.compile(r" À "), " To "),
+)
+
+
+def _fr_labels(chunk):
+    """`chunk` (a copy) with the parser's English structural labels in French."""
+    for field, pattern, french in _FR_LABELS:
+        if chunk.get(field):
+            chunk[field] = pattern.sub(french, chunk[field])
+    return chunk
+
+
+def section_en(section):
+    """The English twin's section label for a French one (see _SECTION_EN)."""
+    for pattern, english in _SECTION_EN:
+        section = pattern.sub(english, section)
+    return section
+
+
+def _frenchify(chunks, code):
+    """French twins of `chunks`: a '-fr-' id, lang 'fr', the French citation
+    style and URL, the parser's structural labels in French (_FR_LABELS), and
+    `section_en`, the English twin's section label, for pairing the two."""
+    out = []
+    for c in chunks:
+        c = dict(c)
+        c["id"] = c["id"].replace(f"{code}-", f"{code}-fr-", 1)
+        c["lang"] = "fr"
+        c["citation"] = (c["citation"]
+                         .replace(", s. ", ", art. ")
+                         .replace(", Schedule", ", annexe"))
+        # the French site cites a regulation as DORS- and a C.R.C. chapter as ch.,
+        # as _fr_xml_url does: /fra/reglements/SOR-... and _c._ return 404, so
+        # every French regulation link was dead until 2026-10-12
+        c["source_url"] = (c["source_url"]
+                           .replace("/eng/acts/", "/fra/lois/")
+                           .replace("/eng/regulations/SOR-", "/fra/reglements/DORS-")
+                           .replace("/eng/regulations/", "/fra/reglements/")
+                           .replace("_c._", "_ch._"))
+        c = _fr_labels(c)
+        c["section_en"] = section_en(c.get("section") or "")
+        out.append(c)
+    return out
+
+
+def ingest_fr(code, force=False, allow_shrink=False):
+    src = SOURCES[code]
+    RAW_DIR.mkdir(parents=True, exist_ok=True)
+    dest = RAW_DIR / f"{code}-fr.xml"
+    if not dest.exists() or force:
+        url = _fr_xml_url(src)
+        print(f"  downloading {url}")
+        _download_xml(url, dest)
+    chunks = _frenchify(_parse_or_evict(dest, code), code)
+    out = PROCESSED_DIR / f"{code}-fr.json"
+    if not write_corpus(out, chunks, indent=2, allow_shrink=allow_shrink):
+        raise RuntimeError(f"{code}: refused to write a collapsed corpus")
+    print(f"  {len(chunks)} articles -> {out.name}")
+    return chunks
+
+
+def main():
+    force = "--force" in sys.argv
+    french = "--fr" in sys.argv
+    allow_shrink = "--allow-shrink" in sys.argv
+    only = [a for a in sys.argv[1:] if not a.startswith("-")]
+    codes = only or list(SOURCES)
+    failures = []
+    for code in codes:
+        if code not in SOURCES:
+            print(f"  SKIP {code}: not a known source")
+            continue
+        try:
+            if french:
+                print(f"Ingesting {code} (français)...")
+                ingest_fr(code, force=force, allow_shrink=allow_shrink)
+            else:
+                ingest(code, force=force, allow_shrink=allow_shrink)
+        except Exception as exc:
+            failures.append((code, exc))
+            print(f"  FAILED {code}: {type(exc).__name__}: {exc}")
+    print(f"\nDone: {len(codes) - len(failures)}/{len(codes)} ingested.")
+    for code, exc in failures:
+        print(f"  FAILED {code}: {exc}")
+    # One collapsed Act must not pass as a clean run: refresh.py and the
+    # RUNBOOK chain this into `&& py -m canlex.embed`.
+    sys.exit(1 if failures else 0)
+
+
+if __name__ == "__main__":
+    main()

canlex/irb_guidelines.py

@@ -0,0 +1,165 @@
+"""Ingest the IRB Chairperson's Guidelines the border workflow actually meets.
+
+Guideline 2 (Detention) is what Immigration Division members apply at every
+48-hour/7-day/30-day review an officer attends; Guidelines 3 (child
+claimants) and 8 (vulnerable persons) govern how detained or referred minors
+and vulnerable people are handled. Soft law: issued under IRPA
+s. 159(1)(h), members are expected to apply them or explain why not --
+ingested as doc_type='memorandum' (guidance), like the D-memoranda.
+
+Pages are public HTML on irb-cisr.gc.ca (same host caselaw.py already
+parses for jurisprudential guides).
+
+    python -m canlex.irb_guidelines
+"""
+import re
+import sys
+
+from bs4 import BeautifulSoup
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "irb_guidelines"
+
+GUIDELINES = [
+    {"num": "2", "file": "guideline2",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir02.aspx",
+     "title": "Detention"},
+    {"num": "3", "file": "guideline3",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir03.aspx",
+     "title": "Child Refugee Claimants: Procedural and Evidentiary Issues"},
+    {"num": "8", "file": "guideline8",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir08.aspx",
+     "title": "Accessibility to IRB Proceedings — Procedural Accommodations "
+              "and Substantive Considerations"},
+    # The three not loaded in the first pass: gender-based claims (4), how
+    # an RPD hearing is prepared and run (7), and SOGIESC claims (9) --
+    # what a port-of-entry claim referred to the RPD meets next.
+    {"num": "4", "file": "guideline4",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir04.aspx",
+     "title": "Gender Considerations in Proceedings Before the IRB"},
+    {"num": "7", "file": "guideline7",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir07.aspx",
+     "title": "Preparation and Conduct of a Hearing in the Refugee "
+              "Protection Division"},
+    {"num": "9", "file": "guideline9",
+     "url": "https://irb.gc.ca/en/legal-policy/policies/Pages/GuideDir09.aspx",
+     "title": "Proceedings Before the IRB Involving Sexual Orientation, "
+              "Gender Identity and Expression, and Sex Characteristics"},
+]
+
+# "Effective date: May 31, 2021" (or amended/revised) in the page prose.
+_EFFECTIVE = re.compile(
+    r"(?:effective|amended|revised)[^.:\n]*:?\s*((?:January|February|March|"
+    r"April|May|June|July|August|September|October|November|December)\s+"
+    r"\d{1,2},\s+\d{4})", re.I)
+_MONTHS = {m: i for i, m in enumerate(
+    ("January", "February", "March", "April", "May", "June", "July",
+     "August", "September", "October", "November", "December"), start=1)}
+
+
+def _iso(prose):
+    m = re.match(r"(\w+)\s+(\d{1,2}),\s+(\d{4})", prose or "")
+    if not m or m.group(1) not in _MONTHS:
+        return ""
+    return f"{m.group(3)}-{_MONTHS[m.group(1)]:02d}-{int(m.group(2)):02d}"
+
+
+def _sections(main):
+    """Yield (label, heading, text) per numbered heading in the page body."""
+    heads = main.find_all(["h2", "h3"])
+    for i, h in enumerate(heads):
+        heading = _norm(h.get_text())
+        m = re.match(r"(\d+(?:\.\d+)*)\.?\s+(.*)", heading)
+        label = m.group(1) if m else str(i + 1)
+        title = m.group(2) if m else heading
+        parts = []
+        for sib in h.next_siblings:
+            if getattr(sib, "name", None) in ("h2", "h3"):
+                break
+            if getattr(sib, "get_text", None):
+                parts.append(sib.get_text("\n"))
+        body = "\n".join(p for p in (_norm(x) for x in
+                                     "\n".join(parts).split("\n")) if p)
+        if len(body) > 80:
+            yield label, title, body
+
+
+def build(allow_shrink=False, force=False):
+    chunks, failures = [], []
+    for g in GUIDELINES:
+        try:
+            html = fetch_cached(g["url"], RAW / f"{g['file']}.html",
+                                pause=1.0,
+                                force=force).decode("utf-8", "replace")
+        except Exception as exc:
+            print(f"  !! Guideline {g['num']}: fetch failed "
+                  f"({type(exc).__name__}: {exc})")
+            failures.append(f"IRB-G{g['num']}")
+            continue
+        soup = BeautifulSoup(html, "html.parser")
+        main = soup.find("main") or soup
+        eff = _EFFECTIVE.search(main.get_text())
+        effective = _iso(eff.group(1)) if eff else ""
+        n = 0
+        for label, title, body in _sections(main):
+            pieces = split_lines(body, 1800) if len(body) > 2400 else [body]
+            total = len(pieces)
+            for k, piece in enumerate(pieces, start=1):
+                suffix = "" if total == 1 else f"-p{k}"
+                note = "" if total == 1 else f" (part {k} of {total})"
+                chunks.append({
+                    "id": f"irbg-{g['num']}-{label.replace('.', '-')}{suffix}",
+                    "doc_type": "memorandum",
+                    "act_code": f"IRB-G{g['num']}",
+                    "act_short": f"IRB Guideline {g['num']}",
+                    "act_name": f"IRB Chairperson's Guideline {g['num']} — "
+                                f"{g['title']}",
+                    "section": f"G{g['num']} s. {label}",
+                    "marginal_note": title,
+                    "part": g["title"],
+                    "division": "",
+                    "heading": "",
+                    "text": piece,
+                    "history": "",
+                    "last_amended": effective,
+                    "current_to": effective,
+                    "citation": f"Chairperson's Guideline {g['num']}, "
+                                f"s. {label} ({title}){note}",
+                    "source_url": g["url"],
+                })
+                n += 1
+        if not n:
+            # Fetched, parsed to nothing: the IRB page template moved.
+            print(f"  !! Guideline {g['num']}: no content parsed; "
+                  f"keeping stored copy")
+            failures.append(f"IRB-G{g['num']}")
+            continue
+        print(f"  Guideline {g['num']} ({g['title'][:40]}): {n} chunks, "
+              f"effective {effective or '?'}")
+    out = PROCESSED_DIR / "irb_guidelines.json"
+    chunks, preserved = preserve_dropped(chunks, stored_chunks(out),
+                                         key=lambda c: c["act_code"],
+                                         only=set(failures))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} guideline(s) "
+              f"that failed this run: {', '.join(preserved)}")
+    uniquify_ids(chunks)
+    if not write_corpus(out, chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"irb_guidelines: {len(chunks)} chunks -> {out.name}")
+    return chunks
+
+
+def main():
+    # `is not False` rather than truthiness: a first-ever run that writes an
+    # empty corpus succeeded, a refused write did not.
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv) is not False
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/irb_papers.py

@@ -0,0 +1,480 @@
+"""Ingest the IRB Legal Services papers the s. 36 screening chain leans on.
+
+The equivalency pairings cite Hill, Brannson, Li, Steward, Lei, Ward, Masasi
+and the rest through the IRB Legal Services papers -- *Removal Order
+Appeals*, ch. 7 (Criminal Grounds for Removal), ch. 8 (Criminal
+Equivalency) and ch. 9 (Discretionary Jurisdiction), and *Sponsorship
+Appeals*, ch. 2 (Criminal Refusals) -- which synthesise the Federal Court
+line on equivalency, on what counts as a conviction, on the ID's and IAD's
+jurisdiction over the criminality grounds and on the IAD's humanitarian
+discretion after a s. 36 removal order. Public PDFs on irb-cisr.gc.ca;
+ingested as doc_type='memorandum' (persuasive legal commentary by the
+Board's own counsel, not law), like the Chairperson's Guidelines.
+
+Archived. The IRB's legal-concepts index pages (RoaAmr.aspx, SpoPar.aspx)
+title every chapter 'ARCHIVED' and carry the Government of Canada archived-
+content notice (kept for reference, not updated since archiving); both
+pages were last modified 2018-07-06. Every chunk records this: status
+'archived', `archived` (that date), `archive_index` (the index page) and a
+`history` line the renderer prints under the text. The papers are dated
+2008/2009, so they predate later case law and amendments.
+
+Parsing reads PyMuPDF's font information, not bare text lines (a plain-text
+parse titled 38 of SPO ch. 2's 39 chunks 'January 1, 2008' and three ROA
+ch. 8 sections with sentence fragments):
+- running heads -- series name, page number, 'Legal Services', the paper's
+  date, '<title> – Ch. N' -- are bold lines in the footer band, dropped
+  (with a text filter for the same strings on any bold line), so the date
+  is never a section title;
+- a heading is a short bold block of body size or larger: Arial Bold for a
+  section, Times Bold for a sub-heading ('Steps in Analysis', titled
+  '<section>: <sub-heading>'); a bold ALL-CAPS line ('GENERAL PRINCIPLES')
+  is a part heading and goes in `division`;
+- a bold candidate that reads as a fragment -- ends in ':', ',' or ';' or a
+  function word, or has a footnote number glued to a word ('Dayan,19') --
+  stays body text;
+- footnotes (10 pt, opened by a small marker) become '[fn N] ...' paragraphs
+  after the page's text, and superscript references read '[fn N]'.
+
+Section numbers are CanLex's (the PDFs number nothing). ROA ch. 8's were in
+use before this parser, so its PAPERS row pins them (`pins`): a pinned
+heading keeps its number, a heading found between two pins takes a decimal
+(§ 1.1, § 20.2), and the old § 9 and § 12 -- page-break fragments, not
+headings -- are folded into the sections the PDF puts them in (§ 8, § 11).
+Unpinned papers number their sections 1, 2, 3, ...
+
+    python -m canlex.irb_papers [--force] [--allow-shrink]
+"""
+import re
+import sys
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "irb_papers"
+_BASE = "https://www.irb-cisr.gc.ca/en/legal-policy/legal-concepts/Documents/"
+_INDEX = "https://www.irb-cisr.gc.ca/en/legal-policy/legal-concepts/Pages/"
+# 'Date modified' of both index pages, which title every chapter ARCHIVED.
+ARCHIVED = "2018-07-06"
+
+# ROA ch. 8's section numbers as first ingested (2026-09-10), in document
+# order. The second 'Committing an Offence Outside Canada' is a real second
+# heading (under 'Evidentiary Matters').
+_ROA8_PINS = [
+    ("Introduction", 1),
+    ("Equating the Foreign Offence to a Canadian Federal Statute", 2),
+    ("Determining Equivalency", 3),
+    ("Determining Equivalency: Leading Federal Court Dicta", 4),
+    ("Where the Foreign Law is Available", 5),
+    ("Where the Foreign Law is Available: Steps in Analysis", 6),
+    ("Where the Foreign Law Is Not Available", 7),
+    ("Malum in se Offences", 8),
+    ("Committing an Offence Outside Canada", 10),
+    ("Foreign Convictions", 11),
+    ("Committing an Offence Outside Canada", 13),
+    ("Relevant Time for Determining Inadmissibility", 14),
+    ("Section 44 Report as a Limiting Factor", 15),
+    ("Discharges and Pardons", 16),
+    ("Effect of a Discharge", 17),
+    ("Effect of a Pardon", 18),
+    ("Rehabilitation", 19),
+    ("Offences Committed Outside Canada by Minors", 20),
+    ("Cases", 21),
+]
+
+
+def _roa(ch, title, **extra):
+    return dict({
+        "code": f"IRB-LS-ROA{ch}", "file": f"RoaAmr{ch:02d}_e.pdf",
+        "short": f"IRB Legal Services, Removal Order Appeals ch. {ch}",
+        "name": f"IRB Legal Services, Removal Order Appeals, Chapter {ch} -- "
+                f"{title} (January 1, 2009; archived by the IRB)",
+        "title": title, "label": f"ROA ch. {ch}", "dated": "2009-01-01",
+        "index": _INDEX + "RoaAmr.aspx"}, **extra)
+
+
+PAPERS = [
+    _roa(7, "Criminal Grounds for Removal"),
+    _roa(8, "Criminal Equivalency", pins=_ROA8_PINS),
+    _roa(9, "Discretionary Jurisdiction"),
+    {"code": "IRB-LS-SPO2", "file": "SpoPar02_e.pdf",
+     "short": "IRB Legal Services, Sponsorship Appeals ch. 2",
+     "name": "IRB Legal Services, Sponsorship Appeals, Chapter 2 -- "
+             "Criminal Refusals (January 1, 2008; archived by the IRB)",
+     "title": "Criminal Refusals", "label": "SPO ch. 2",
+     "dated": "2008-01-01", "index": _INDEX + "SpoPar.aspx"},
+]
+
+_MONTH_NAMES = ("January", "February", "March", "April", "May", "June",
+                "July", "August", "September", "October", "November",
+                "December")
+_MONTHS = {m: i for i, m in enumerate(_MONTH_NAMES, start=1)}
+_M = "|".join(_MONTH_NAMES)
+_DATE = re.compile(rf"({_M}) (\d{{1,2}}), (\d{{4}})")
+# The running-head pieces the papers print on every page, each usually its
+# own line: series name, 'Legal Services', page number, the paper's date,
+# '<title> – Ch. N'. A line made only of these pieces is a running head.
+_PIECE = (rf"(?:Removal Order Appeals|Sponsorship Appeals|Legal Services|"
+          rf"\d{{1,3}}|(?:{_M}) \d{{1,2}}, \d{{4}}|"
+          rf"[A-Z][^\n]{{2,80}}?\s[–-]\s?Ch\.\s?\d{{1,2}})")
+RUNNING_LINE = re.compile(rf"^{_PIECE}(?:\s+{_PIECE})*$", re.I)
+
+_FOOTER = 90          # pt from the page bottom: the running-head band
+_TITLE_SIZE = 14      # 'Chapter Eight' / the chapter title (16 pt)
+_HEAD_SIZE = 11.5     # headings are 12 pt; running heads 11 pt
+_NOTE_SIZE = 10.8     # footnote text is 10 pt; body 11-12 pt
+_SUP_SIZE = 8.5       # superscript references 8 pt; footnote markers 6.5 pt
+_FUNCTION_END = re.compile(
+    r"\b(?:of|the|and|or|to|in|for|a|an|with|by|on|at|from|that|as|is|are|"
+    r"under|where|which)$", re.I)
+_GLUED_NUMBER = re.compile(r"[a-z][,;:]\d{1,3}\b")     # 'Dayan,19', 'follows:1'
+_TERMINATED = re.compile(r"[.?!:;\"”’)\]]$")
+_FN_REF = re.compile(r"\s*\[fn \d+\]")
+
+
+def is_fragment(text):
+    """A bold line that is a sentence fragment or a running date, not a
+    heading: ends in ':', ',' or ';' or a function word, carries a footnote
+    number glued to a word, or is a bare date."""
+    t = text.strip()
+    return (t.endswith((":", ",", ";")) or bool(_FUNCTION_END.search(t))
+            or bool(_GLUED_NUMBER.search(t)) or bool(_DATE.fullmatch(t)))
+
+
+def _is_bold(span):
+    return "Bold" in span["font"] or bool(span.get("flags", 0) & 16)
+
+
+def _is_glyph(span):
+    return any(f in span["font"] for f in ("Wingdings", "Symbol"))
+
+
+def line_record(spans):
+    """One PDF line as {text, size, bold, arial, fn, bullet}, or None when
+    blank. Superscript digits become '[fn N]' in the text; a line opened by
+    a small digit and otherwise at footnote size is a footnote (fn = N).
+    Pure (takes PyMuPDF span dicts), for tests."""
+    live = [s for s in spans if s["text"].strip()]
+    if not live:
+        return None
+    top = max(s["size"] for s in spans)
+
+    def sup(s):
+        return (s["text"].strip().isdigit() and s["size"] <= _SUP_SIZE
+                and (s["size"] < 0.8 * top or len(live) == 1))
+
+    fn, bullet, parts = None, False, []
+    rest = [s for s in live[1:]]
+    if sup(live[0]) and max((s["size"] for s in rest), default=0) <= _NOTE_SIZE:
+        fn = live[0]["text"].strip()
+    for s in spans:
+        if s is live[0] and fn is not None:
+            continue
+        text = s["text"]
+        if s is live[0] and (_is_glyph(s) or text.strip() in ("•", "▪")) \
+                and len(text.strip()) == 1:
+            bullet = True
+            continue
+        if text.strip() and sup(s):
+            parts.append(f"[fn {text.strip()}]")
+        else:
+            parts.append(text)
+    words = [s for s in live if not sup(s) and not _is_glyph(s)]
+    text = _norm("".join(parts))
+    text = re.sub(r"(\S)\[fn", r"\1 [fn", text)
+    return {"text": text,
+            "size": max((s["size"] for s in words), default=top),
+            "bold": bool(words) and all(_is_bold(s) for s in words),
+            "arial": bool(words) and "Arial" in words[0]["font"],
+            "fn": fn, "bullet": bullet}
+
+
+def pdf_pages(path):
+    """Per page, the line records in reading order, each with its block
+    number, top y and the page height."""
+    import fitz   # PyMuPDF
+    pages = []
+    with fitz.open(path) as doc:
+        for page in doc:
+            lines = []
+            blocks = page.get_text("dict", sort=True)["blocks"]
+            for bno, block in enumerate(blocks):
+                for line in block.get("lines", []):
+                    rec = line_record(line["spans"])
+                    if rec:
+                        rec.update(block=bno, y=line["bbox"][1],
+                                   h=page.rect.height)
+                        lines.append(rec)
+            pages.append(lines)
+    return pages
+
+
+def _join(texts):
+    """Line texts into one paragraph, de-hyphenating line-end breaks."""
+    out = ""
+    for t in texts:
+        t = _norm(t)
+        if not t:
+            continue
+        if out.endswith("-") and len(out) > 1 and out[-2].isalpha() \
+                and t[:1].islower():
+            out = out[:-1] + t
+        else:
+            out = f"{out} {t}" if out else t
+    return out
+
+
+def _iso(m):
+    return f"{m.group(3)}-{_MONTHS[m.group(1)]:02d}-{int(m.group(2)):02d}"
+
+
+def items(pages):
+    """(items, date): the paper as a flat list of ('part', title),
+    ('h', level, title), ('p', text) and ('fn', number, text), plus the
+    date its running head carries ('' if none). Pure given line records."""
+    out, date, in_cases, pending = [], "", False, ""
+
+    def last_p():
+        """Index of the last paragraph when only footnotes follow it."""
+        for i in range(len(out) - 1, -1, -1):
+            if out[i][0] == "p":
+                return i
+            if out[i][0] != "fn":
+                return None
+        return None
+
+    def add_p(text, first_on_page):
+        i = last_p() if first_on_page else None
+        if i is not None and text[:1].islower() \
+                and not _TERMINATED.search(_FN_REF.sub("", out[i][1])):
+            out[i] = ("p", f"{out[i][1]} {text}")    # continues across the page
+        else:
+            out.append(("p", text))
+
+    def flush_pending():
+        nonlocal pending
+        if pending:
+            out.append(("h", 2, pending.rstrip(" –-")))
+            pending = ""
+
+    for lines in pages:
+        blocks = []
+        for ln in lines:
+            if ln["bold"] and (ln["y"] >= ln["h"] - _FOOTER
+                               or RUNNING_LINE.match(ln["text"])):
+                m = _DATE.search(ln["text"])
+                if m and not date:
+                    date = _iso(m)
+                continue
+            if ln["size"] >= _TITLE_SIZE:
+                continue
+            if not ln["text"] and ln["fn"] is None:
+                continue        # a glyph alone (the steps diagrams' arrows)
+            if blocks and blocks[-1][0]["block"] == ln["block"]:
+                blocks[-1].append(ln)
+            else:
+                blocks.append([ln])
+        first_on_page = True
+        for blk in blocks:
+            text = _FN_REF.sub("", _join(ln["text"] for ln in blk))
+            heading = (all(ln["bold"] and ln["fn"] is None for ln in blk)
+                       and len(blk) <= 3 and len(text) <= 160
+                       and text[:1].isupper()
+                       and (min(ln["size"] for ln in blk) >= _HEAD_SIZE
+                            or text.upper() == "CASES"))
+            # The table of cases: 'CASES' in bold (ch. 7, 8), 'Cases' in
+            # plain type (ch. 9) -- a block of that one word either way.
+            if text.upper() == "CASES" and len(blk) == 1:
+                flush_pending()
+                out.append(("h", 2, "Cases"))
+                in_cases = True
+                continue
+            if heading and text.endswith(("–", "-")):
+                pending = text                  # 'Rehabilitation –' / 'Burden of Proof'
+                continue
+            if heading and pending:
+                text, pending = f"{pending} {text}", ""
+            if heading and text.isupper() and len(text) > 4:
+                out.append(("part", text.title()))
+                continue
+            if heading and not is_fragment(text):
+                out.append(("h", 2 if blk[0]["arial"] else 3, text))
+                continue
+            flush_pending()
+            body = []
+            for k, ln in enumerate(blk):
+                if ln["fn"] is not None and not ln["text"] and not (
+                        k + 1 < len(blk) and blk[k + 1]["size"] < _NOTE_SIZE):
+                    # a lone superscript on its own line (a numbered step's
+                    # reference), not a footnote opener: keep it inline
+                    ref = f"[fn {ln['fn']}]"
+                    if body:
+                        body[-1] = f"{body[-1]} {ref}"
+                    elif out and out[-1][0] == "p":
+                        out[-1] = ("p", f"{out[-1][1]} {ref}")
+                    continue
+                if ln["fn"] is not None:
+                    if body:
+                        add_p(_join(body), first_on_page)
+                        first_on_page, body = False, []
+                    out.append(("fn", ln["fn"], ln["text"]))
+                elif (not in_cases and not ln["bold"]
+                      and ln["size"] < _NOTE_SIZE
+                      and any(it[0] == "fn" for it in out)):
+                    if out[-1][0] == "fn":             # footnote text / continuation
+                        n, t = out[-1][1], out[-1][2]
+                        out[-1] = ("fn", n, _join([t, ln["text"]]))
+                    else:                              # note continued from the last page
+                        j = max(i for i, it in enumerate(out) if it[0] == "fn")
+                        out.append(("fn", out[j][1], ln["text"]))
+                else:
+                    if ln["bullet"] and body:
+                        add_p(_join(body), first_on_page)
+                        first_on_page, body = False, []
+                    body.append(("- " if ln["bullet"] else "") + ln["text"])
+            if body:
+                add_p(_join(body), first_on_page)
+                first_on_page = False
+    flush_pending()
+    return out, date
+
+
+def sections(its, pins=None):
+    """(sections, unused_pins): [{num, title, division, body}] from items().
+    A level-3 heading is titled '<section>: <sub-heading>'. Numbers follow
+    `pins` ([(title, number)] in document order) when given -- unpinned
+    headings between pins take decimals -- else run 1, 2, 3, ..."""
+    queue, has_pins = list(pins or []), bool(pins)
+    secs, parent, division = [], "", ""
+    state = {"int": 0, "sub": 0}
+
+    def number(title):
+        if queue and queue[0][0] == title:
+            state["int"], state["sub"] = queue.pop(0)[1], 0
+            return str(state["int"])
+        if has_pins:
+            state["sub"] += 1
+            return f"{state['int']}.{state['sub']}"
+        state["int"] += 1
+        return str(state["int"])
+
+    def open_section(title):
+        secs.append({"num": number(title), "title": title,
+                     "division": division, "body": []})
+
+    for it in its:
+        if it[0] == "part":
+            division = it[1]
+        elif it[0] == "h":
+            title = it[2]
+            if it[1] == 2:
+                parent = title
+            elif parent:
+                title = f"{parent}: {title}"
+            open_section(title)
+        else:
+            if not secs:
+                open_section("Introduction")
+            secs[-1]["body"].append(it[1] if it[0] == "p"
+                                    else f"[fn {it[1]}] {it[2]}")
+    for s in secs:
+        s["body"] = "\n".join(s["body"])
+    return secs, queue
+
+
+def history(paper, dated):
+    page = paper["index"].rsplit("/", 1)[-1]
+    return (f"ARCHIVED by the IRB: its legal-concepts index ({page}, "
+            f"modified {ARCHIVED}) marks this paper archived -- kept for "
+            f"reference, not updated since. Paper dated {dated}.")
+
+
+def chunks_for(paper, secs, dated):
+    """Corpus chunks for one paper's sections. Pure, for tests."""
+    out = []
+    for s in secs:
+        body = s["body"].strip()
+        if not body:
+            continue
+        pieces = split_lines(body, 1800) if len(body) > 2400 else [body]
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            out.append({
+                "id": f"{paper['code'].lower()}-{s['num']}{suffix}",
+                "doc_type": "memorandum",
+                "act_code": paper["code"],
+                "act_short": paper["short"],
+                "act_name": paper["name"],
+                "section": f"{paper['label']} § {s['num']}",
+                "marginal_note": s["title"],
+                "part": paper["title"],
+                "division": s["division"],
+                "heading": "",
+                "text": piece,
+                "history": history(paper, dated),
+                "last_amended": dated,
+                "current_to": dated,
+                "status": "archived",
+                "archived": ARCHIVED,
+                "archive_index": paper["index"],
+                "citation": f"{paper['short']}, § {s['num']} "
+                            f"({s['title']}){note}",
+                "source_url": _BASE + paper["file"],
+            })
+    return out
+
+
+def build(allow_shrink=False, force=False):
+    chunks, failures = [], []
+    for paper in PAPERS:
+        try:
+            pdf = RAW / paper["file"]
+            fetch_cached(_BASE + paper["file"], pdf, pause=1.0, force=force)
+            its, date = items(pdf_pages(pdf))
+        except Exception as exc:
+            print(f"  !! {paper['code']}: fetch/parse failed "
+                  f"({type(exc).__name__}: {exc})")
+            failures.append(paper["code"])
+            continue
+        secs, unused = sections(its, paper.get("pins"))
+        if unused:
+            print(f"  !! {paper['code']}: pinned headings not found "
+                  f"(section numbers may have moved): "
+                  f"{', '.join(t for t, _ in unused)}")
+        # The running head carries the paper's date; the PAPERS row is the
+        # fallback when the template changes.
+        dated = date or paper.get("dated", "")
+        got = chunks_for(paper, secs, dated)
+        if not got:
+            print(f"  !! {paper['code']}: no content parsed; keeping stored copy")
+            failures.append(paper["code"])
+            continue
+        chunks += got
+        print(f"  {paper['code']} ({paper['title']}): "
+              f"{sum(1 for s in secs if s['body'].strip())} sections, "
+              f"{len(got)} chunks, dated {dated or '?'}")
+    out = PROCESSED_DIR / "irb_papers.json"
+    chunks, preserved = preserve_dropped(chunks, stored_chunks(out),
+                                         key=lambda c: c["act_code"],
+                                         only=set(failures))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} paper(s) "
+              f"that failed this run: {', '.join(preserved)}")
+    uniquify_ids(chunks)
+    if not write_corpus(out, chunks, indent=2, allow_shrink=allow_shrink):
+        return False
+    print(f"irb_papers: {len(chunks)} chunks -> {out.name}")
+    return chunks
+
+
+def main():
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv) is not False
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/loganalyze.py

@@ -0,0 +1,381 @@
+"""Mine the MCP call log: what people ask CanLex, and where it falls short.
+
+Two record shapes are read:
+  calls.jsonl  -- one record per tool call (canlex/calllog.py, 2026-09-16):
+                  CANLEX_LOG_DIR/calls.jsonl on a server with CANLEX_LOG_DIR
+                  set, and logs/YYYY-MM-DD/<uuid>.jsonl in the private dataset
+                  the Space pushes to (CANLEX_LOG_DATASET).
+  queries.jsonl -- the older search-only log (server._log_query until
+                  2026-09-16), read as canlex_search_legislation calls.
+
+    py -m canlex.loganalyze                     # CANLEX_LOG_DIR: calls.jsonl, queries.jsonl
+    py -m canlex.loganalyze PATH [PATH ...]     # files, or folders searched for *.jsonl
+    py -m canlex.loganalyze --dataset Beemer0/canlex-call-log
+                                                # token: CANLEX_LOG_TOKEN (this process's
+                                                # environment, else the Windows user
+                                                # environment variable of that name), else
+                                                # asked for with hidden input in a terminal
+    py -m canlex.loganalyze --days 7 ...        # only the calls of the last 7 days (with
+                                                # --dataset, only those days are downloaded)
+
+It reports calls per tool and per day, the most common question texts and
+offence strings, no-result and weak-match searches (the only unbiased source
+of corpus and synonym gaps, since the eval set is authored by the
+maintainer), NOT SETTLED screening results, and errors.
+
+Privacy: the arguments are what users typed -- offences, dates, places, and
+sometimes personal facts about a traveller. Treat every log as sensitive: keep
+the dataset private, never commit a log (data/logs/ is gitignored), and do not
+paste this report anywhere public. A --dataset download goes to a temporary
+folder that is deleted afterwards unless --out names one.
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+# Mirror server.HEDGE_THRESHOLD's intent without importing the server: a query
+# whose best result scored below this is a weak match worth reviewing.
+_WEAK = 0.72
+
+SEARCH_TOOL = "canlex_search_legislation"
+# CANLEX_RESULT branch codes and rehabilitation statuses the screening tools
+# print as NOT SETTLED (mirrored, not imported, like _WEAK above). 'unknown'
+# is the server's no-pairing-matched branch (server.py, grouped with the
+# open branches in screening.py); 'undetermined' is rehab's 'an open fact
+# decides a limb' status, which the memo prints as NOT SETTLED. Both were
+# counted as settled until 2026-09-21. tests/test_calllog.py pins this list
+# against the codes the server and engine emit.
+OPEN_BRANCHES = ("depends", "tie", "tran-open", "limb-open", "unknown")
+NOT_SETTLED_STATUS = "not-settled"
+NOT_SETTLED_STATUSES = (NOT_SETTLED_STATUS, "undetermined")
+# canlex_transborder on a composed release without the IRPR s. 19(1)(c)-(g)
+# table: printed 'NOT ASSESSED -- the ... table is held for review', its own
+# label, so the report counts it on its own line, not under NOT SETTLED.
+HELD_BRANCH = "table-held"
+OFFENCE_KEYS = ("offense", "offence")
+TOP = 20
+
+
+def _load(path):
+    rows = []
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rows.append(json.loads(line))
+        except ValueError:
+            continue
+    return rows
+
+
+def log_files(paths):
+    """The .jsonl files named, or found (recursively) under the folders named."""
+    out = []
+    for p in map(Path, paths):
+        if p.is_dir():
+            out += sorted(f for f in p.rglob("*.jsonl") if f.is_file())
+        elif p.is_file():
+            out.append(p)
+    return out
+
+
+def _stamp(ts):
+    """A record's ts as ISO 8601 text (a legacy record's is epoch seconds)."""
+    if isinstance(ts, (int, float)) and not isinstance(ts, bool):
+        return (datetime.datetime.fromtimestamp(ts, datetime.timezone.utc)
+                .isoformat(timespec="seconds").replace("+00:00", "Z"))
+    return str(ts or "")
+
+
+def _day(ts):
+    return _stamp(ts)[:10] or "?"
+
+
+def normalize(row):
+    """A call record, or None. A legacy queries.jsonl row (a 'query', no
+    'tool') becomes a canlex_search_legislation call."""
+    if not isinstance(row, dict):
+        return None
+    if isinstance(row.get("tool"), str):
+        if not isinstance(row.get("arguments"), dict):
+            row = dict(row, arguments={})
+        return row
+    if "query" not in row:
+        return None
+    return {"ts": row.get("ts"), "tool": SEARCH_TOOL,
+            "arguments": {k: row.get(k) for k in ("query", "top_k", "act", "doc_type")},
+            "ok": True, "n_results": row.get("n_results"),
+            "confidence": row.get("confidence"),
+            "top_citations": row.get("top_citations") or [], "legacy": True}
+
+
+def load_calls(paths):
+    """(calls, files read) from the files and folders named."""
+    files = log_files(paths)
+    calls = []
+    for f in files:
+        calls += [c for c in map(normalize, _load(f)) if c is not None]
+    calls.sort(key=lambda c: _stamp(c.get("ts")))
+    return calls, files
+
+
+def _text(value):
+    return " ".join(str(value).split()).strip().lower() if value is not None else ""
+
+
+def offences(call):
+    """The offence strings a screening call's arguments carry."""
+    args = call.get("arguments") or {}
+    out = [args[k] for k in OFFENCE_KEYS if isinstance(args.get(k), str)]
+    entries = args.get("entries")
+    if isinstance(args.get("entries_json"), str):
+        try:
+            entries = json.loads(args["entries_json"])
+        except ValueError:
+            entries = None
+    for e in entries if isinstance(entries, list) else []:
+        if isinstance(e, dict):
+            out += [e[k] for k in OFFENCE_KEYS if isinstance(e.get(k), str)]
+    return [t for t in map(_text, out) if t]
+
+
+def not_settled(call):
+    """True when a screening result is NOT SETTLED: an open branch (a tie,
+    a fact-dependent branch, no pairing matched), or a rehabilitation status
+    that is not settled or undetermined."""
+    result = call.get("result")
+    if not isinstance(result, dict):
+        return False
+    if result.get("branch") in OPEN_BRANCHES:
+        return True
+    for key in ("rehab_status", "status"):
+        value = result.get(key)
+        values = value if isinstance(value, list) else [value]
+        if any(v in NOT_SETTLED_STATUSES for v in values):
+            return True
+    return False
+
+
+def held_table(call):
+    """True when canlex_transborder answered NOT ASSESSED because the
+    composed release ships without the s. 19(1)(c)-(g) table."""
+    result = call.get("result")
+    return isinstance(result, dict) and result.get("branch") == HELD_BRANCH
+
+
+def _summary(call, width=70):
+    args = call.get("arguments") or {}
+    offs = offences(call)
+    text = (args.get("query") if isinstance(args.get("query"), str)
+            else "; ".join(offs) if offs
+            else json.dumps(args, ensure_ascii=False, sort_keys=True))
+    text = " ".join(str(text).split())
+    return text if len(text) <= width else text[:width - 3] + "..."
+
+
+def since(days, now=None):
+    """The ISO 8601 UTC cutoff for 'the last `days` days', or None for no cutoff."""
+    if not days:
+        return None
+    now = now or datetime.datetime.now(datetime.timezone.utc)
+    return (now - datetime.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S")
+
+
+def window_patterns(days, now=None):
+    """snapshot_download allow_patterns for the dataset's logs/YYYY-MM-DD/ folders
+    that can hold a call from the last `days` days (every folder with no cutoff)."""
+    if not days:
+        return ["logs/*"]
+    now = now or datetime.datetime.now(datetime.timezone.utc)
+    return [f"logs/{(now - datetime.timedelta(days=n)).strftime('%Y-%m-%d')}/*"
+            for n in range(days + 1)]
+
+
+def report(paths, out=None, days=None, now=None):
+    out = out or sys.stdout
+    paths = list(paths) if isinstance(paths, (list, tuple)) else [paths]
+    calls, files = load_calls(paths)
+    cutoff = since(days, now)
+    if cutoff:
+        calls = [c for c in calls if _stamp(c.get("ts")) >= cutoff]
+
+    def say(line=""):
+        print(line, file=out)
+
+    where = ", ".join(str(f) for f in files[:3]) + (f" and {len(files) - 3} more"
+                                                    if len(files) > 3 else "")
+    if not calls:
+        say(f"No logged tool calls in {where or ', '.join(map(str, paths))}"
+            + (f" in the last {days} day(s)." if cutoff else "."))
+        return calls
+    legacy = sum(1 for c in calls if c.get("legacy"))
+    if cutoff:
+        say(f"Window: the last {days} day(s), since {cutoff}Z.")
+    span = sorted({_day(c.get("ts")) for c in calls})
+    say(f"{len(calls)} logged tool calls, {span[0]} to {span[-1]}, from {len(files)} "
+        f"file(s): {where}" + (f" ({legacy} from the legacy search log)" if legacy else ""))
+    dropped = sum(c.get("dropped_before") or 0 for c in calls
+                  if isinstance(c.get("dropped_before"), int))
+    if dropped:
+        say(f"{dropped} further call(s) were not recorded (the log's queue was full).")
+
+    per_tool = Counter(c.get("tool") for c in calls)
+    errors_by_tool = Counter(c.get("tool") for c in calls if c.get("ok") is False)
+    say("\n== Calls per tool ==")
+    for tool, n in per_tool.most_common():
+        errs = errors_by_tool.get(tool)
+        say(f"  {n:>5}  {tool}" + (f"  ({errs} error(s))" if errs else ""))
+
+    say("\n== Calls per day ==")
+    for day, n in sorted(Counter(_day(c.get("ts")) for c in calls).items()):
+        say(f"  {day}  {n:>5}")
+
+    searches = [c for c in calls if c.get("tool") == SEARCH_TOOL]
+    questions = Counter(t for t in (_text((c.get("arguments") or {}).get("query"))
+                                    for c in searches) if t)
+    say(f"\n== Most common questions ({SEARCH_TOOL} query; {len(questions)} distinct) ==")
+    for q, n in questions.most_common(TOP):
+        say(f"  {n:>3}x  {q}")
+
+    offence_counts = Counter(o for c in calls for o in offences(c))
+    say(f"\n== Most common offence strings (screening tools; {len(offence_counts)} distinct) ==")
+    for o, n in offence_counts.most_common(TOP):
+        say(f"  {n:>3}x  {o}")
+
+    ok_searches = [c for c in searches if c.get("ok") is not False]
+    no_result = [c for c in ok_searches if c.get("n_results") == 0]
+    weak = sorted((c for c in ok_searches
+                   if isinstance(c.get("confidence"), (int, float))
+                   and c["confidence"] < _WEAK and c.get("n_results")),
+                  key=lambda c: c["confidence"])
+    say(f"\n== No-result searches ({len(no_result)}) -- likely corpus or vocabulary gaps ==")
+    for c in no_result[:25]:
+        say(f"  {_summary(c)}")
+    say(f"\n== Weak matches (confidence < {_WEAK}; {len(weak)}) -- candidate eval "
+        f"questions / synonyms ==")
+    for c in weak[:25]:
+        top = (c.get("top_citations") or ["?"])[0]
+        say(f"  {c['confidence']:.3f}  {_summary(c):70}  -> {top}")
+
+    unsettled = [c for c in calls if not_settled(c)]
+    say(f"\n== NOT SETTLED screening results ({len(unsettled)}) ==")
+    for c in unsettled[-25:]:
+        result = c.get("result") or {}
+        what = ", ".join(f"{k}={result[k]}" for k in ("branch", "rehab_status", "status")
+                         if result.get(k) is not None)
+        needs = result.get("needs") or []
+        say(f"  {_day(c.get('ts'))}  {c.get('tool')}  {what}  -- {_summary(c, 50)}")
+        if needs:
+            say(f"      needs: {needs[0]}")
+
+    held = [c for c in calls if held_table(c)]
+    say(f"\n== NOT ASSESSED (held table) transborder results ({len(held)}) -- "
+        f"the s. 19(1)(c)-(g) table is held for review on the deployment ==")
+    for c in held[-25:]:
+        say(f"  {_day(c.get('ts'))}  {c.get('tool')}  -- {_summary(c, 50)}")
+
+    errors = [c for c in calls if c.get("ok") is False]
+    say(f"\n== Errors ({len(errors)}) ==")
+    for (tool, err), n in Counter((c.get("tool"), c.get("error") or "?")
+                                  for c in errors).most_common():
+        say(f"  {n:>5}  {tool}  {err}")
+    for c in errors[-10:]:
+        say(f"      {_stamp(c.get('ts'))}  {c.get('tool')}  {c.get('error') or '?'}  -- {_summary(c, 60)}")
+    return calls
+
+
+def download_dataset(repo_id, token, dest, days=None, now=None):
+    """The dataset's logs/ folder (only the last `days` days' folders when
+    given), downloaded into `dest`; returns its path."""
+    from huggingface_hub import snapshot_download
+    return snapshot_download(repo_id=repo_id, repo_type="dataset", token=token,
+                             allow_patterns=window_patterns(days, now), local_dir=str(dest))
+
+
+def user_env_token():
+    """CANLEX_LOG_TOKEN as a Windows *user* environment variable, read from the
+    registry, or "". An unattended run (the weekly check) inherits its parent
+    app's environment, which predates a variable set later in Windows' own
+    'Edit environment variables for your account' dialog; reading the registry
+    finds it without restarting anything. Deliberately NOT huggingface_hub's
+    saved login (`hf auth login` / HF_TOKEN): that token may be broad, and the
+    log should be read only with one scoped to the log dataset."""
+    try:
+        import winreg
+        with winreg.OpenKey(winreg.HKEY_CURRENT_USER, "Environment") as key:
+            value, _kind = winreg.QueryValueEx(key, "CANLEX_LOG_TOKEN")
+        return str(value or "").strip()
+    except Exception:
+        return ""
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(
+        prog="python -m canlex.loganalyze",
+        description="Report on the CanLex call log (calls.jsonl, legacy queries.jsonl, or "
+                    "the private log dataset). The log may hold personal facts: keep it "
+                    "private.")
+    ap.add_argument("paths", nargs="*",
+                    help="log files, or folders searched for *.jsonl (default: CANLEX_LOG_DIR)")
+    ap.add_argument("--dataset", metavar="REPO",
+                    help="download logs/ from this private Hugging Face dataset repo "
+                         "(token from the CANLEX_LOG_TOKEN environment variable -- this "
+                         "process's, else the Windows user variable -- else asked for with "
+                         "hidden input in a terminal)")
+    ap.add_argument("--days", metavar="N", type=int,
+                    help="report only the calls of the last N days (with --dataset, only "
+                         "those days' folders are downloaded)")
+    ap.add_argument("--out", metavar="DIR",
+                    help="with --dataset: keep the download in DIR (default: a temporary "
+                         "folder, deleted afterwards)")
+    args = ap.parse_args(argv)
+
+    if args.dataset:
+        token = (os.environ.get("CANLEX_LOG_TOKEN") or "").strip() or user_env_token()
+        if not token and sys.stdin is not None and sys.stdin.isatty():
+            # asked for with hidden input, never typed on a command line: a
+            # shell's history file keeps what is typed there (PowerShell 5.1's
+            # PSReadLine 2.0 saves every line, in plain text)
+            import getpass
+            try:
+                token = getpass.getpass(f"Hugging Face token that can read {args.dataset} "
+                                        "(input hidden): ").strip()
+            except (EOFError, KeyboardInterrupt):
+                token = ""
+        if not token:
+            print("No token: run this in a terminal to be asked for a Hugging Face token "
+                  f"that can read {args.dataset}, or set CANLEX_LOG_TOKEN as a Windows user "
+                  "environment variable (Windows search: 'Edit environment variables for "
+                  "your account'). Never type the token itself on a command line.",
+                  file=sys.stderr)
+            sys.exit(2)
+        if args.out:
+            folder = download_dataset(args.dataset, token, Path(args.out), args.days)
+            report([folder] + list(args.paths), days=args.days)
+            return
+        with tempfile.TemporaryDirectory(prefix="canlex-call-log-",
+                                         ignore_cleanup_errors=True) as tmp:
+            folder = download_dataset(args.dataset, token, Path(tmp), args.days)
+            report([folder] + list(args.paths), days=args.days)
+        return
+
+    paths = list(args.paths)
+    if not paths:
+        log_dir = Path(os.environ.get("CANLEX_LOG_DIR", "."))
+        paths = [p for p in (log_dir / "calls.jsonl", log_dir / "queries.jsonl") if p.exists()]
+    if not log_files(paths):
+        print(f"No call log at {', '.join(map(str, paths)) or os.environ.get('CANLEX_LOG_DIR', '.')}."
+              " Set CANLEX_LOG_DIR on the server (and pass the path here), or use "
+              "--dataset.", file=sys.stderr)
+        sys.exit(1)
+    report(paths, days=args.days)
+
+
+if __name__ == "__main__":
+    main()

canlex/narrative.py

@@ -0,0 +1,731 @@
+"""The DRAFT s. 44(1) narrative skeleton in the ENF 5 s. 13.1 form, English
+and French (roadmap T4.6; track A step A15, 2026-09-15 -- HELD FOR REVIEW).
+
+ENF 5 s. 13.1 (2025-02-20) says that in applying A36(2)(b) 'it is not
+sufficient to state that the person has been convicted of an offence', and
+gives the narrative's form. This module renders that form from what the
+screening chain holds, and from nothing else:
+
+- ENF 5's frames, verbatim (EN_FRAME, FR_FRAME -- tests/test_narrative.py
+  pins every fragment to enf-5-13-1-p2 and enf-fr-5-13-1-p2/-p3);
+- the IRPA paragraph the chain proposes, with its words verbatim (English:
+  the memo's _ALLEGATION, passed in; French: FR_ALLEGATION, pinned to
+  I-2.5-fr-s36-p1);
+- the Canadian provision from the pairing's optional 'report_provision'
+  (curated JSON, HELD FOR REVIEW; every line verbatim in the corpus in both
+  languages), chosen by the matched limb and the maximum at commission;
+- the maximum in the statute's own words, in each language -- ONE rule for
+  both: the words the provision's own line uses for the term ('not more than
+  10 years' / 'de dix ans'; 'for life' / 'a perpetuite').
+
+What the tool does not hold -- the place and date of issue, the addressee,
+the name and aliases, the city, the conviction date unless given, the
+status evidence, the signature -- stays [CONFIRM: ...]; a value the tool
+proposes is bracketed with what to confirm. Where the chain is not settled
+(a tie, the limb, the offence date, the conviction question) every reading
+gets its own allegation, each with its condition: nothing open is printed
+as settled. A maximum at commission that differs from the current
+provision's (Tran: a curated history row, or a provision not in force at
+commission) prints [CONFIRM], because the former text is not in the corpus.
+A composed release on the approved curated JSON has no report_provision: the
+provision and maximum then stay [CONFIRM], and the block says the provisions
+are held for review.
+"""
+import re
+
+from canlex.screening import LIFE, _NUM_WORDS, limb_max
+
+HEAD = "## DRAFT s. 44(1) narrative skeleton (ENF 5 s. 13.1): verify every bracket"
+INTRO = (
+    "DRAFT -- not a report, and nothing in it is found for you. ENF 5 s. 13.1 (2025-02-20) "
+    "sets the elements of an A44(1) report: 'it is not sufficient to state that the person "
+    "has been convicted of an offence'. The frames below are ENF 5's words; '(drafted)' marks "
+    "CanLex's wording, for which ENF 5 gives no sample (the title, the allegation sentence, "
+    "the act-branch and two-offence narratives). Verify every bracket: [CONFIRM: ...] is a "
+    "fact CanLex does not hold, and [value -- ... CONFIRM] a value CanLex proposes from the "
+    "chain above. The maximum is the statute's own words for the maximum at commission, in "
+    "each language.")
+MULTI = ("ENF 5 s. 13.4: 'it is generally recommended that the officer writes a separate "
+         "report for each allegation' -- one allegation per skeleton below; where a condition "
+         "is shown, the chain has not settled which applies.")
+HELD_NOTE = ("The Canadian provisions (the pairings' 'report_provision') are HELD FOR REVIEW "
+             "and not in this deployment: the provision and the maximum stay [CONFIRM].")
+
+# ENF 5 s. 13.1 (2025-02-20), verbatim: the English sample (enf-5-13-1-p2)
+# and the French one (enf-fr-5-13-1-p2/-p3). The brackets the sample fills
+# are added by the renderer.
+EN_FRAME = {
+    "heading": "This report is based on the following information that the above-named individual:",
+    "aka": "Also known as",
+    "status_fn": ("-Is not a Canadian citizen; is not a permanent resident of Canada; is "
+                  "not a registered Indian under the Indian Act;"),
+    "convicted": "…has been convicted of an offence; namely,",
+    "on_or_about": "on or about",
+    "at_or_near": "at or near",
+    "equivalence": ("This offence, if committed in Canada, would constitute an offence that "
+                    "may be punishable by way of indictment under"),
+    "maximum": "and for which a maximum term of imprisonment",
+    "imposed": "may be imposed.",
+}
+FR_FRAME = {
+    "heading": ("Le présent rapport est fondé sur les renseignements suivants indiquant que "
+                "la personne susnommée :"),
+    "aka": "Aussi connu sous le ou les noms suivants :",
+    "status_fn": ("-n’est pas un citoyen canadien; n’est pas un résident permanent du Canada; "
+                  "n’est pas un Indien inscrit en vertu de la Loi sur les Indiens;"),
+    "convicted": "… a été reconnu coupable d’une infraction; nommément",
+    "on_or_about": "le ou aux environs du",
+    "at_or_near": "à ou près de",
+    "equivalence": ("Cette infraction, commise au Canada, constituerait une infraction "
+                    "punissable par mise en accusation en vertu"),
+    "maximum": "et pour laquelle une peine maximale d’emprisonnement",
+    "imposed": "peut être imposée.",
+}
+FRAME_SOURCE = {"en": ("enf-5-13-1-p2",), "fr": ("enf-fr-5-13-1-p2", "enf-fr-5-13-1-p3")}
+
+# IRPA s. 36 in French, verbatim (I-2.5-fr-s36-p1): the words of each
+# paragraph the allegation names -- the French twin of the memo's _ALLEGATION.
+FR_ALLEGATION = {
+    "36(1)(b)": ("être déclaré coupable, à l’extérieur du Canada, d’une infraction qui, "
+                 "commise au Canada, constituerait une infraction sous le régime d’une loi "
+                 "fédérale punissable d’un emprisonnement maximal d’au moins dix ans",),
+    "36(1)(c)": ("commettre, à l’extérieur du Canada, une infraction qui, commise au Canada, "
+                 "constituerait une infraction sous le régime d’une loi fédérale punissable "
+                 "d’un emprisonnement maximal d’au moins dix ans",),
+    "36(2)(b)": ("être déclaré coupable, à l’extérieur du Canada, d’une infraction qui, "
+                 "commise au Canada, constituerait une infraction sous le régime d’une loi "
+                 "fédérale punissable par mise en accusation",),
+    "36(2)(b)-two": ("être déclaré coupable, à l’extérieur du Canada,",
+                     "de deux infractions qui ne découlent pas des mêmes faits et qui, "
+                     "commises au Canada, constitueraient des infractions sous le régime de "
+                     "toute loi fédérale"),
+    "36(2)(c)": ("commettre, à l’extérieur du Canada, une infraction qui, commise au Canada, "
+                 "constituerait une infraction sous le régime d’une loi fédérale punissable "
+                 "par mise en accusation",),
+}
+FR_PARAGRAPH = {"36(1)(b)": "36(1)b)", "36(1)(c)": "36(1)c)", "36(2)(b)": "36(2)b)",
+                "36(2)(c)": "36(2)c)"}
+IRPA_EN = "the Immigration and Refugee Protection Act"
+IRPA_FR = "de la Loi sur l’immigration et la protection des réfugiés"
+
+# The narratives ENF 5 gives no sample for: an act (s. 36(1)(c)/(2)(c)) and
+# the two-offence limb of s. 36(2)(b). CanLex's wording, built on ENF 5's
+# sentence and the statute's words; printed '(drafted)'.
+EN_ACT = ("…has committed an act outside Canada; namely, [{act}] on or about [{date}] at or "
+          "near [{place}], that is an offence in the place where it was committed, namely "
+          "[CONFIRM: the foreign provision], and that, if committed in Canada, would "
+          "constitute an offence that may be punishable by way of indictment under [{prov}] "
+          "and for which a maximum term of imprisonment [{maxw}] may be imposed. (drafted)")
+FR_ACT = ("… a commis, à l’extérieur du Canada, une infraction, nommément [{act}], le ou aux "
+          "environs du [{date}] à ou près de [{place}], qui constitue une infraction dans le "
+          "lieu où elle a été commise, nommément [CONFIRMER : la disposition étrangère], et "
+          "qui, commise au Canada, constituerait une infraction punissable par mise en "
+          "accusation en vertu [{prov}] et pour laquelle une peine maximale d’emprisonnement "
+          "[{maxw}] peut être imposée. (rédigé par CanLex)")
+EN_TWO = ("…has been convicted of two offences not arising out of a single occurrence; "
+          "namely, {offences}. These offences, if committed in Canada, would constitute "
+          "offences under an Act of Parliament, namely {provisions}. (drafted)")
+FR_TWO = ("… a été reconnu coupable de deux infractions qui ne découlent pas des mêmes faits; "
+          "nommément {offences}. Ces infractions, commises au Canada, constitueraient des "
+          "infractions sous le régime de toute loi fédérale, nommément {provisions}. "
+          "(rédigé par CanLex)")
+
+# CanLex's labels and bracket notes, per language.
+TEXT = {
+    "en": {
+        "block": "### English",
+        "place_date": "Place and date of issue: [CONFIRM: place of issue], [CONFIRM: date of issue]",
+        "addressee": ("To: [CONFIRM: the Minister of Public Safety or the Minister of IRCC -- "
+                      "ENF 5 allows either]"),
+        "title": ("Report under subsection 44(1) of the Immigration and Refugee Protection Act "
+                  "(drafted title -- not ENF 5 wording)"),
+        "name": "Name: [CONFIRM: the complete name, correctly spelled]",
+        "aka": "[CONFIRM: other names used, where applicable -- or delete the line]",
+        "signature": ("[CONFIRM: name and signature of the officer who conducted the "
+                      "examination or is otherwise making the report]"),
+        "allegation": ("The above-named individual is, in my opinion, inadmissible under "
+                       "paragraph [{para} -- proposed by CanLex from the chain; CONFIRM] of "
+                       "{irpa}: '{text}'. (drafted)"),
+        "which": "Allegation {k} of {n}{cond}:",
+        "cond": " -- {cond}",
+        "entry": "entry #{n}, ",
+        "status_evidence": ("[CONFIRM: the evidence that the person holds none of these "
+                            "statuses (ENF 5 s. 11.3)]"),
+        "status_assumed": "[CONFIRM: status not given -- CanLex assumed a foreign national]",
+        "status_pr": ("-[CONFIRM: a permanent resident's status line -- ENF 5 s. 13.1 gives "
+                      "none (unsourced draft: 'Is a permanent resident of Canada; is not a "
+                      "Canadian citizen; is not a registered Indian under the Indian Act;')]"),
+        "offence": ("{offence} -- the offence as recorded; CONFIRM the offence of conviction "
+                    "(not the charge) against the certificate of conviction"),
+        "act": "CONFIRM: the act -- the record gives '{offence}'",
+        "date": ("CONFIRM: the date{given} -- ENF 5 s. 13.1 does not say whether it is the "
+                 "conviction or the offence date; Appendix E asks for the Conviction Date"),
+        "act_date": "CONFIRM: the date of the act{given}",
+        "given_conv": "; conviction date given: {d}",
+        "given_off": "; offence date given: {d}",
+        "place": "CONFIRM: the city or town{given}",
+        "given_place": " -- the record gives {p}",
+        "proposed": "{cite} -- proposed from the curated pairing {pid} (statute line {chunk}); CONFIRM",
+        "others": "; or, only where the facts are: {alts}",
+        "alt": "{when}: {cite}",
+        "open": "CONFIRM which: {alts} -- the record's words do not say",
+        "open_alt": "{cite} ({when})",
+        "history": ("CONFIRM: the provision in force at commission -- the maximum then was "
+                    "{at} (CanLex's reading, {basis}); its text is not in the corpus. Today's "
+                    "provision, for reference: {cite}"),
+        "history_nif": ("CONFIRM: the provision in force at commission -- today's provision "
+                        "came into force on {frm}, after the offence; the maximum at "
+                        "commission on CanLex's reading: {at}; the former text is not in the "
+                        "corpus. Today's provision, for reference only: {cite}"),
+        "history_higher": ("CONFIRM: the provision in force at commission -- the curated "
+                           "history names it: {label}; its text is not in the corpus. Today's "
+                           "provision, for reference only (not the provision at commission): "
+                           "{cite}"),
+        "later": "; not listed, as not in force at commission: {alts}",
+        "later_alt": "{cite} (in force from {frm})",
+        "held": ("CONFIRM: the provision -- this deployment holds no report provision for "
+                 "{pid} (HELD FOR REVIEW); the pairing names {canadian}"),
+        "inchoate": ("CONFIRM: the inchoate provision (Criminal Code ss. 463-465) with the "
+                     "object offence's provision{ref}"),
+        "inchoate_ref": " -- the object offence: {cite}",
+        "none": ("CONFIRM: the provision -- no curated report provision of {pid} carries {at}; "
+                 "the pairing names {canadian}"),
+        "derived": ("{cite} -- proposed from the curated pairing {pid} (statute line {chunk}); "
+                    "CONFIRM, with the object offence's provision"),
+        "max_words": "{words} -- the statute's words ({chunk}); the maximum {basis}; CONFIRM",
+        "max_open": "CONFIRM: the provision decides -- {alts}",
+        "max_open_alt": "{words} ({cite})",
+        "max_history": ("CONFIRM: {at} -- the maximum at commission from the curated history; "
+                        "the statute's words then are not in the corpus"),
+        "max_both": ("CONFIRM -- EACH READING, NOT SETTLED: {then} at commission (Tran read "
+                     "strictly; {label}; the statute's words then are not in the corpus) | "
+                     "{now} on methodology (b) (the lesser maximum in force now, under today's "
+                     "provision)"),
+        "max_nif": ("CONFIRM: {at} -- CanLex's reading: the provision was not in force at "
+                    "commission, and the statute's words then are not in the corpus"),
+        "max_reading": "CONFIRM: {at} -- CanLex's maximum {basis}",
+        "max_inchoate": "CONFIRM: {at} -- CanLex's reading of the inchoate rule, from the object offence",
+        "max_derived": "CONFIRM: the maximum is derived from the object offence's",
+        "max_unknown": "CONFIRM: the maximum at commission",
+        "basis_date": "at commission (the offence date given)",
+        "basis_current": ("now in force -- the offence date was not given, so the maximum at "
+                          "commission is not confirmed"),
+        "basis_bound": "at commission, bounded by the {what} date given",
+        "basis_other": "at commission",
+        "two_offence": "[{offence} -- {entry}CONFIRM] on or about [{date}] at or near [{place}]",
+        "two_entry": "entry #{n}; ",
+        "two_other": "[CONFIRM: the other offence] on or about [CONFIRM: the date] at or near [CONFIRM: the place]",
+        "two_prov": "[{cite} -- proposed from the curated pairing {pid}; CONFIRM]",
+        "two_prov_confirm": "[CONFIRM: the provision for {what}]",
+        "and": " and ",
+        "life": "for life",
+        "years": "{n} years",
+    },
+    "fr": {
+        "block": "### Français",
+        "place_date": "Lieu et date : [CONFIRMER : lieu], [CONFIRMER : date]",
+        "addressee": ("Destinataire : [CONFIRMER : ministre de la Sécurité publique ou ministre "
+                      "d’IRCC -- l’ENF 5 permet l’un ou l’autre]"),
+        "title": ("Rapport établi en vertu du paragraphe 44(1) de la Loi sur l’immigration et la "
+                  "protection des réfugiés (titre rédigé par CanLex -- pas le libellé de l’ENF 5)"),
+        "name": "Nom : [CONFIRMER : nom complet, épelé correctement]",
+        "aka": "[CONFIRMER : autres noms utilisés, le cas échéant -- ou supprimer la ligne]",
+        "signature": ("[CONFIRMER : nom et signature de l’agent qui a mené l’examen ou qui a "
+                      "rédigé le rapport]"),
+        "allegation": ("À mon avis, la personne susnommée est interdite de territoire au titre de "
+                       "l’alinéa [{para} -- proposé par CanLex d’après la chaîne; CONFIRMER] "
+                       "{irpa} : « {text} ». (rédigé par CanLex)"),
+        "which": "Allégation {k} sur {n}{cond} :",
+        "cond": " -- hypothèse (note de CanLex, en anglais) : {cond}",
+        "entry": "inscription no {n}, ",
+        "status_evidence": ("[CONFIRMER : la preuve que la personne n’a aucun de ces statuts "
+                            "(ENF 5, section 11.3)]"),
+        "status_assumed": "[CONFIRMER : statut non fourni -- CanLex a supposé un étranger]",
+        "status_pr": ("-[CONFIRMER : ligne de statut d’un résident permanent -- l’ENF 5, section "
+                      "13.1, n’en donne aucune (ébauche non sourcée : « est un résident permanent "
+                      "du Canada; n’est pas un citoyen canadien; n’est pas un Indien inscrit en "
+                      "vertu de la Loi sur les Indiens; »)]"),
+        "offence": ("{offence} -- infraction inscrite au dossier; CONFIRMER la déclaration de "
+                    "culpabilité (et non l’accusation initiale) sur le certificat"),
+        "act": "CONFIRMER : l’acte -- le dossier indique « {offence} »",
+        "date": ("CONFIRMER : la date{given} -- l’ENF 5, section 13.1, ne précise pas s’il "
+                 "s’agit de la date de la déclaration de culpabilité ou de celle de "
+                 "l’infraction; l’annexe E demande la date de la déclaration de culpabilité"),
+        "act_date": "CONFIRMER : la date de l’acte{given}",
+        "given_conv": "; déclaration de culpabilité : {d}",
+        "given_off": "; infraction : {d}",
+        "place": "CONFIRMER : la ville{given}",
+        "given_place": " -- le dossier indique {p}",
+        "proposed": ("{cite} -- proposé d’après l’appariement organisé {pid} (ligne de la loi "
+                     "{chunk}); CONFIRMER"),
+        "others": "; ou, seulement si les faits le montrent (note en anglais) : {alts}",
+        "alt": "{when} : {cite}",
+        "open": "CONFIRMER laquelle : {alts} -- les termes du dossier ne le disent pas",
+        "open_alt": "{cite} ({when})",
+        "history": ("CONFIRMER : la disposition en vigueur au moment de l’infraction -- la peine "
+                    "maximale était alors de {at} (lecture de CanLex); son texte n’est pas dans "
+                    "le corpus. La disposition actuelle, à titre indicatif : {cite}"),
+        "history_nif": ("CONFIRMER : la disposition en vigueur au moment de l’infraction -- la "
+                        "disposition actuelle est entrée en vigueur le {frm}, après l’infraction; "
+                        "peine maximale au moment de l’infraction selon la lecture de CanLex : "
+                        "{at}; l’ancien texte n’est pas dans le corpus. La disposition actuelle, "
+                        "à titre indicatif seulement : {cite}"),
+        "history_higher": ("CONFIRMER : la disposition en vigueur au moment de l’infraction -- "
+                           "l’historique organisé la désigne (note en anglais) : {label}; son "
+                           "texte n’est pas dans le corpus. La disposition actuelle, à titre "
+                           "indicatif seulement (ce n’est pas celle en vigueur au moment de "
+                           "l’infraction) : {cite}"),
+        "later": "; non retenue(s), car pas en vigueur au moment de l’infraction : {alts}",
+        "later_alt": "{cite} (en vigueur depuis le {frm})",
+        "held": ("CONFIRMER : la disposition -- ce déploiement ne contient aucune disposition de "
+                 "rapport pour {pid} (EN ATTENTE D’EXAMEN)"),
+        "inchoate": ("CONFIRMER : la disposition relative à la tentative, à la complicité ou au "
+                     "complot (Code criminel, art. 463 à 465), avec celle de l’infraction "
+                     "visée{ref}"),
+        "inchoate_ref": " -- l’infraction visée : {cite}",
+        "none": ("CONFIRMER : la disposition -- aucune disposition de rapport organisée de "
+                 "{pid} ne porte ce maximum ({at})"),
+        "derived": ("{cite} -- proposé d’après l’appariement organisé {pid} (ligne de la loi "
+                    "{chunk}); CONFIRMER, avec la disposition de l’infraction visée"),
+        "max_words": ("{words} -- les mots de la loi ({chunk}); la peine maximale {basis}; "
+                      "CONFIRMER"),
+        "max_open": "CONFIRMER : selon la disposition -- {alts}",
+        "max_open_alt": "{words} ({cite})",
+        "max_history": ("CONFIRMER : {at} -- la peine maximale au moment de l’infraction selon "
+                        "l’historique organisé; le texte de la loi alors en vigueur n’est pas "
+                        "dans le corpus"),
+        "max_both": ("CONFIRMER -- CHAQUE LECTURE, NON TRANCHÉ : {then} au moment de "
+                     "l’infraction (arrêt Tran lu strictement; note en anglais : {label}; le "
+                     "texte de la loi alors en vigueur n’est pas dans le corpus) | {now} selon "
+                     "la méthode (b) (la peine maximale moins sévère actuellement en vigueur, "
+                     "selon la disposition actuelle)"),
+        "max_nif": ("CONFIRMER : {at} -- lecture de CanLex : la disposition n’était pas en "
+                    "vigueur au moment de l’infraction, et le texte de la loi alors en vigueur "
+                    "n’est pas dans le corpus"),
+        "max_reading": "CONFIRMER : {at} -- la peine maximale selon CanLex, {basis}",
+        "max_inchoate": ("CONFIRMER : {at} -- lecture de CanLex de la règle applicable à la "
+                         "tentative ou à la complicité, d’après l’infraction visée"),
+        "max_derived": "CONFIRMER : la peine maximale découle de celle de l’infraction visée",
+        "max_unknown": "CONFIRMER : la peine maximale au moment de l’infraction",
+        "basis_date": "au moment de l’infraction (date de l’infraction fournie)",
+        "basis_current": ("actuellement en vigueur -- la date de l’infraction n’a pas été "
+                          "fournie; la peine maximale au moment de l’infraction n’est pas "
+                          "confirmée"),
+        "basis_bound": "au moment de l’infraction, bornée par la date fournie ({what})",
+        "basis_other": "au moment de l’infraction",
+        "two_offence": ("[{offence} -- {entry}CONFIRMER] le ou aux environs du "
+                        "[{date}] à ou près de [{place}]"),
+        "two_entry": "inscription no {n}; ",
+        "two_other": ("[CONFIRMER : l’autre infraction] le ou aux environs du [CONFIRMER : la "
+                      "date] à ou près de [CONFIRMER : le lieu]"),
+        "two_prov": "[{cite} -- proposé d’après l’appariement organisé {pid}; CONFIRMER]",
+        "two_prov_confirm": "[CONFIRMER : la disposition pour {what}]",
+        "and": " et ",
+        "life": "à perpétuité",
+        "years": "{n} ans",
+    },
+}
+_BOUND_WHAT = {"en": {"conviction": "conviction", "completion": "sentence-completion"},
+               "fr": {"conviction": "déclaration de culpabilité",
+                      "completion": "fin de la peine"}}
+NO_SKELETON = {
+    "none": "No skeleton: no Canadian federal equivalent -- no s. 36 allegation on this matter.",
+    "pr": ("No skeleton: a permanent resident is not reached by s. 36(2), and this matter is "
+           "below the s. 36(1) threshold."),
+    "single-summary": ("No skeleton: a single summary-only equivalent grounds nothing -- "
+                       "s. 36(2)(b) reaches two such offences not arising out of a single "
+                       "occurrence."),
+    "not-settled": ("No skeleton yet: the Canadian equivalent is not settled ({why}) -- "
+                    "settle it and re-run."),
+    "unknown": "No skeleton: no Canadian equivalent identified.",
+    "canada": ("No skeleton for entry #{n}: a conviction in Canada (s. 36(1)(a)/(2)(a)) -- ENF 5 "
+               "s. 13.1's sample is for a conviction outside Canada; draft from the "
+               "certificate of conviction."),
+    "canada_one": ("No skeleton: a conviction in Canada (s. 36(1)(a)/(2)(a)) -- ENF 5 s. 13.1's "
+                   "sample is for a conviction outside Canada; draft from the certificate of "
+                   "conviction."),
+    "occurrence": ("No skeleton yet for the summary-only entries: the single-occurrence "
+                   "question (s. 36(2)(b)) is open -- both readings above."),
+    # the review of batch 1, round 4 (RP-04): the record's words reach content held for
+    # review and not in this deployment -- nothing is drafted from it
+    "held": ("No skeleton: the record's words reach CanLex content held for review and not in "
+             "this deployment -- no allegation is drafted until that content is approved or the "
+             "disposition is confirmed from the court record."),
+    # disposition roadmap T1.5 (HELD): the act branch after an acquittal
+    "act-not-settled": ("No skeleton for s. {para}: after an acquittal or a finding of not guilty, "
+                        "whether the act branch may be used on the same conduct is NOT SETTLED -- "
+                        "POLICY CAVEAT ({label}), both readings in step 1."),
+    # the review of batch 1, round 3 (MW-08): a routed programme that does not state its
+    # underlying termination (a CPL 160.50 sealing)
+    "act-not-settled-routed": ("No skeleton for s. {para}: if the underlying termination was an "
+                               "acquittal ({question}), whether the act branch may be used on the same "
+                               "conduct is NOT SETTLED -- POLICY CAVEAT ({label}), both readings in "
+                               "step 1."),
+}
+
+# The words a provision's line uses for the term, per language (one rule).
+_EN_TERM = re.compile(r"not (?:more than|exceeding) (?:\d+|[a-z]+) years(?: less a day)?"
+                      r"|up to (?:\d+|[a-z]+) years|imprisonment for life|life imprisonment",
+                      re.I)
+_FR_TERM = re.compile(r"(?:maximale? |^\((?:[ivx]+|[a-z](?:\.\d)?)\) )de ((?:\d+|[a-zé]+) ans"
+                      r"(?: moins un jour)?)|(à perpétuité)")
+# one table for the English number words (audit follow-up 2026-09-21): the
+# one maxima_stated reads, so the two parsers cannot drift apart
+_EN_NUM = _NUM_WORDS
+_FR_NUM = {"un": 1, "deux": 2, "trois": 3, "quatre": 4, "cinq": 5, "six": 6, "sept": 7,
+           "huit": 8, "neuf": 9, "dix": 10, "onze": 11, "douze": 12, "treize": 13,
+           "quatorze": 14}
+
+
+def term_words(line, lang="en"):
+    """The statute's own words for the maximum term in one provision line
+    ('not more than 10 years'; 'de dix ans'; life: 'for life' / 'à
+    perpétuité'), or None when the line states no term (summary conviction,
+    a maximum derived from another offence). The first term the line states
+    is the provision's general one (s. 271(a): 10 years; 14 only for a
+    complainant under 16 -- the item's 'when' says so)."""
+    if lang == "fr":
+        m = _FR_TERM.search((line or "").strip())
+        if not m:
+            return None
+        return "à perpétuité" if m.group(2) else "de " + m.group(1)
+    m = _EN_TERM.search(line or "")
+    if not m:
+        return None
+    t = m.group(0)
+    return "for life" if "life" in t.lower() else t
+
+
+def term_years(words, lang="en"):
+    """The years a term_words() phrase states (LIFE for life), or None."""
+    if not words:
+        return None
+    if "life" in words or "perpétuité" in words:
+        return LIFE
+    num = _FR_NUM if lang == "fr" else _EN_NUM
+    m = re.search(r"(\d+|[a-zé]+) (?:years|ans)", words)
+    if not m:
+        return None
+    g = m.group(1)
+    return int(g) if g.isdigit() else num.get(g)
+
+
+def fr_cite(fr):
+    """The French citation after 'en vertu': 'de l’alinéa ...', 'du paragraphe ...'."""
+    first = (fr or "").split(" ", 1)[0]
+    return ("de l’" if first in ("alinéa", "article") else "du ") + fr
+
+
+def provisions(pairing):
+    return [i for i in (pairing or {}).get("report_provision") or [] if isinstance(i, dict)]
+
+
+def in_force_at(item, latest):
+    """False when a report provision came into force ('in_force_from') after
+    the last day the offence can have been committed on this reading --
+    'latest', an ISO date, or None when no date bounds it (wave-4 verifier,
+    2026-09-15: a provision not in force at commission is never proposed)."""
+    f = item.get("in_force_from")
+    return not (f and latest and str(latest) < f)
+
+
+def choose(pairing, reading, limb_candidates=()):
+    """Which curated report provision(s) fit one reading of a pairing:
+    {"kind": ..., "proposed", "others", "items", "at", "now", "later",
+    "from", "higher"}. kind: 'proposed' (the provision whose line carries the
+    maximum at commission; a primary first), 'open' (candidate limbs with
+    different provisions -- the record's words do not say which), 'history'
+    (the maximum at commission differs from today's provision: a Tran history
+    row, or a provision not in force at commission -- 'from' is the date it
+    came into force, and today's provision is named for reference only,
+    never as the provision at commission), 'inchoate' (an attempt,
+    accessory, counselling or conspiracy reading), 'derived' (a provision
+    whose maximum is the object offence's), 'none' (no provision carries
+    this maximum), 'held' (the pairing has no report_provision in this
+    deployment). reading: the server's _narrative_reading -- 'latest' (the
+    last day the offence can have been committed on it) filters out the
+    provisions not yet in force ('later'); 'higher' is the curated history
+    row whose maximum at commission is HIGHER than the one the reading
+    applies (methodology (b)): the maximum then prints both readings."""
+    allp = provisions(pairing)
+    at = reading.get("max_years")
+    out = {"kind": "held", "proposed": None, "others": [], "items": [], "at": at,
+           "now": limb_max(pairing or {}, reading.get("limb")) if pairing else None,
+           "later": [], "from": reading.get("nif_from"), "higher": reading.get("higher")}
+    if not allp:
+        return out
+    items = [i for i in allp if i.get("role") != "two-plus-summary-only"]
+    limb, cands = reading.get("limb"), [c for c in limb_candidates or [] if c]
+    if limb:
+        pool = [i for i in items if not i.get("limbs") or limb in i["limbs"]]
+    elif cands:
+        pool = [i for i in items if not i.get("limbs") or set(i["limbs"]) & set(cands)]
+    else:
+        pool = items
+    latest = reading.get("latest")
+    later = [i for i in pool if not in_force_at(i, latest)]
+    pool = [i for i in pool if in_force_at(i, latest)]
+    out.update(items=pool, later=later)
+    if reading.get("inchoate"):
+        return dict(out, kind="inchoate")
+    if later and not pool:
+        # every provision of this reading came into force after the offence
+        ref = next((i for i in later if i.get("role") == "primary"), later[0])
+        return dict(out, kind="history", proposed=ref, later=[],
+                    **{"from": out["from"] or min(i["in_force_from"] for i in later)})
+    groups = ({frozenset(set(i["limbs"]) & set(cands)) for i in pool if i.get("limbs")}
+              if cands and not limb else set())
+    if len(groups) > 1:
+        return dict(out, kind="open", items=[i for i in pool if i.get("limbs")])
+    derived = [i for i in pool if i.get("derived")]
+    if derived:
+        prop = next((i for i in derived if i.get("role") == "primary"), derived[0])
+        return dict(out, kind="derived", proposed=prop,
+                    others=[i for i in derived if i is not prop])
+    if reading.get("not_in_force") or out["higher"] or (
+            at is not None and out["now"] is not None and at != out["now"]
+            and not reading.get("resolved")):
+        # the maximum at commission is not today's provision's (Tran)
+        same = [i for i in pool if term_years(term_words(i["line"])) == out["now"]]
+        ref = next((i for i in same if i.get("role") == "primary"), (same or pool or [None])[0])
+        return dict(out, kind="history", proposed=ref)
+    match = ([i for i in pool if term_years(term_words(i["line"])) == at]
+             if at is not None else [])
+    if match:
+        prop = next((i for i in match if i.get("role") == "primary"), match[0])
+        return dict(out, kind="proposed", proposed=prop,
+                    others=[i for i in match if i is not prop])
+    return dict(out, kind="none")
+
+
+def _years(mx, lang):
+    if mx is None:
+        return None
+    t = TEXT[lang]
+    if mx >= LIFE:
+        return t["life"]
+    n = int(mx) if float(mx).is_integer() else mx
+    return t["years"].format(n=n)
+
+
+def _basis(a, lang):
+    t = TEXT[lang]
+    b = a.get("basis") or ""
+    if b == "offence-date":
+        return t["basis_date"]
+    if b == "current":
+        return t["basis_current"]
+    if b.startswith("inferred-from-"):
+        what = b[len("inferred-from-"):]
+        return t["basis_bound"].format(what=_BOUND_WHAT[lang].get(what, what))
+    return t["basis_other"]
+
+
+def _cite(i, lang):
+    return i["en"] if lang == "en" else fr_cite(i["fr"])
+
+
+def _chunk(i, lang):
+    return i.get("chunk") if lang == "en" else i.get("chunk_fr")
+
+
+def provision_text(a, lang):
+    """The provision bracket's contents for one allegation."""
+    t, c, p = TEXT[lang], a["choice"], a.get("pairing") or {}
+    pid, canadian = p.get("id", "?"), p.get("canadian_offence", "?")
+    k = c["kind"]
+    later = ""
+    if c.get("later"):
+        later = t["later"].format(alts=" / ".join(
+            t["later_alt"].format(cite=_cite(i, lang), frm=i["in_force_from"])
+            for i in c["later"]))
+    if k in ("proposed", "derived"):
+        i = c["proposed"]
+        s = t[k].format(cite=_cite(i, lang), pid=pid, chunk=_chunk(i, lang))
+        if c["others"]:
+            s += t["others"].format(alts=" / ".join(
+                t["alt"].format(when=o["when"], cite=_cite(o, lang)) for o in c["others"]))
+        return s + later
+    if k == "open":
+        return t["open"].format(alts=" | ".join(
+            t["open_alt"].format(cite=_cite(i, lang), when=i["when"]) for i in c["items"])) + later
+    if k == "history":
+        ref = c["proposed"]
+        cite = _cite(ref, lang) if ref else canadian
+        if c.get("higher"):
+            return t["history_higher"].format(label=c["higher"].get("limb_label") or canadian,
+                                              cite=cite)
+        if c.get("from"):
+            return t["history_nif"].format(frm=c["from"], at=_years(c["at"], lang) or "?",
+                                           cite=cite)
+        return t["history"].format(at=_years(c["at"], lang) or "?", basis=_basis(a, lang),
+                                   cite=cite)
+    if k == "inchoate":
+        ref = next((i for i in c["items"] if i.get("role") == "primary"),
+                   (c["items"] or [None])[0])
+        return t["inchoate"].format(ref=t["inchoate_ref"].format(cite=_cite(ref, lang))
+                                    if ref else "")
+    if k == "none":
+        return t["none"].format(pid=pid, at=_years(c["at"], lang) or "?", canadian=canadian)
+    return t["held"].format(pid=pid, canadian=canadian)
+
+
+def maximum_text(a, lang):
+    """The maximum bracket's contents for one allegation."""
+    t, c = TEXT[lang], a["choice"]
+    k, at = c["kind"], _years(c["at"], lang)
+    if k == "proposed":
+        i = c["proposed"]
+        words = term_words(i["line_fr"] if lang == "fr" else i["line"], lang)
+        return t["max_words"].format(words=words, chunk=_chunk(i, lang), basis=_basis(a, lang))
+    if k == "open":
+        return t["max_open"].format(alts=" | ".join(
+            t["max_open_alt"].format(words=term_words(i["line_fr"] if lang == "fr"
+                                                      else i["line"], lang) or "?",
+                                     cite=_cite(i, lang)) for i in c["items"]))
+    if k == "derived":
+        return t["max_derived"]
+    if k == "history":
+        h = c.get("higher")
+        if h:
+            # methodology (b) applied the lesser maximum now; Tran read strictly
+            # gives the history row's: both, bracketed (principle 5)
+            return t["max_both"].format(then=_years(h.get("at_commission"), lang) or "?",
+                                        now=at or "?", label=h.get("limb_label") or "")
+        if c.get("from") and c["at"] == c["now"]:
+            return t["max_nif"].format(at=at or "?")
+        return t["max_history"].format(at=at or "?")
+    if k == "inchoate":
+        return t["max_inchoate"].format(at=at or "?")
+    if at is not None:
+        return t["max_reading"].format(at=at, basis=_basis(a, lang))
+    return t["max_unknown"]
+
+
+def _date_text(a, lang, act=False):
+    t = TEXT[lang]
+    given = ""
+    if a.get("conviction_date") and not act:
+        given += t["given_conv"].format(d=a["conviction_date"])
+    if a.get("offence_date"):
+        given += t["given_off"].format(d=a["offence_date"])
+    return t["act_date" if act else "date"].format(given=given)
+
+
+def _place_text(a, lang):
+    t = TEXT[lang]
+    return t["place"].format(given=t["given_place"].format(p=a["place"]) if a.get("place") else "")
+
+
+def _allegation_line(a, lang):
+    t = TEXT[lang]
+    key = "36(2)(b)-two" if a["kind"] == "two" else a["paragraph"]
+    if lang == "fr":
+        text, para = " ".join(FR_ALLEGATION[key]), FR_PARAGRAPH[a["paragraph"]]
+        return t["allegation"].format(para=para, irpa=IRPA_FR, text=text)
+    return t["allegation"].format(para=a["paragraph"], irpa=IRPA_EN, text=a["en_text"])
+
+
+def _narrative(a, lang):
+    """The narrative sentence(s) for one allegation, in ENF 5's form."""
+    t = TEXT[lang]
+    f = FR_FRAME if lang == "fr" else EN_FRAME
+    if a["kind"] == "two":
+        offs = []
+        for e in a.get("entries") or []:
+            offs.append(t["two_offence"].format(
+                offence=e["offence"],
+                entry=t["two_entry"].format(n=e["entry"]) if e.get("entry") else "",
+                date=_date_text(e, lang), place=_place_text(e, lang)))
+        while len(offs) < 2:
+            offs.append(t["two_other"])
+        provs = []
+        for e in a.get("entries") or []:
+            s = next((i for i in provisions(e.get("pairing"))
+                      if i.get("role") == "two-plus-summary-only"), None)
+            provs.append(t["two_prov"].format(cite=_cite(s, lang),
+                                              pid=(e.get("pairing") or {}).get("id", "?"))
+                         if s else t["two_prov_confirm"].format(what=e["offence"]))
+        while len(provs) < 2:
+            provs.append(t["two_prov_confirm"].format(what="the other offence" if lang == "en"
+                                                      else "l’autre infraction"))
+        tpl = FR_TWO if lang == "fr" else EN_TWO
+        return [tpl.format(offences=t["and"].join(offs), provisions=t["and"].join(provs))]
+    prov, maxw = provision_text(a, lang), maximum_text(a, lang)
+    if a["kind"] == "act":
+        tpl = FR_ACT if lang == "fr" else EN_ACT
+        return [tpl.format(act=t["act"].format(offence=a["offence"]),
+                           date=_date_text(a, lang, act=True), place=_place_text(a, lang),
+                           prov=prov, maxw=maxw)]
+    return [f"{f['convicted']} [{t['offence'].format(offence=a['offence'])}] "
+            f"{f['on_or_about']} [{_date_text(a, lang)}] {f['at_or_near']} "
+            f"[{_place_text(a, lang)}]. {f['equivalence']} [{prov}] {f['maximum']} "
+            f"[{maxw}] {f['imposed']}"]
+
+
+def _status_line(person, status_given, lang):
+    t = TEXT[lang]
+    f = FR_FRAME if lang == "fr" else EN_FRAME
+    if person == "permanent-resident":
+        return t["status_pr"]
+    if not status_given:
+        return f"{t['status_assumed']} {f['status_fn']}"
+    return f"{f['status_fn']} {t['status_evidence']}"
+
+
+def render(allegations, person, status_given, language="en", notes=(), held=False):
+    """The skeleton block's lines. allegations: from the chain (see the
+    server's _narrative_allegations); notes: 'No skeleton' lines for what
+    grounds no allegation; held: no report_provision in this deployment."""
+    langs = {"en": ("en",), "fr": ("fr",), "both": ("en", "fr")}.get(language, ("en",))
+    if not allegations:
+        return [HEAD] + list(notes)
+    out = [HEAD, INTRO]
+    if held:
+        out.append(HELD_NOTE)
+    if len(allegations) > 1:
+        out.append(MULTI)
+    out += list(notes)
+    n = len(allegations)
+    for lang in langs:
+        t = TEXT[lang]
+        f = FR_FRAME if lang == "fr" else EN_FRAME
+        out += ["", t["block"], t["place_date"], t["addressee"], t["title"], t["name"]]
+        for k, a in enumerate(allegations, 1):
+            if n > 1 or a.get("condition") or a.get("entry"):
+                cond = ((t["entry"].format(n=a["entry"]) if a.get("entry") else "")
+                        + (a.get("condition") or "")).strip().rstrip(",")
+                out.append(t["which"].format(k=k, n=n,
+                                             cond=t["cond"].format(cond=cond) if cond else ""))
+            out.append(_allegation_line(a, lang))
+            out.append(f["heading"])
+            out.append(f"{f['aka']} {t['aka']}")
+            out.append(_status_line(person, status_given, lang))
+            out += _narrative(a, lang)
+        out.append(t["signature"])
+    return out
+
+
+def result_field(allegations, language, held=False):
+    """CANLEX_RESULT 'narrative': each allegation's paragraph, pairing and
+    the provision proposed (None where it stays [CONFIRM])."""
+    rows = []
+    for a in allegations:
+        c = a.get("choice") or {}
+        prop = c.get("proposed") if c.get("kind") in ("proposed", "derived") else None
+        rows.append({"paragraph": a["paragraph"], "kind": a["kind"],
+                     "pairing": (a.get("pairing") or {}).get("id")
+                     if a["kind"] != "two" else [(e.get("pairing") or {}).get("id")
+                                                 for e in a.get("entries") or []],
+                     "entry": a.get("entry"), "condition": a.get("condition"),
+                     "provision": prop["en"] if prop else None,
+                     "provision_status": c.get("kind") if a["kind"] != "two" else "two-offence",
+                     # wave-4 verifier: today's provision came into force after
+                     # the offence; the maximum at commission is higher than
+                     # the one applied (both are bracketed)
+                     "not_in_force_from": c.get("from") if c.get("kind") == "history" else None,
+                     "maximum_readings": ({"at_commission": c["higher"].get("at_commission"),
+                                           "methodology_b": c.get("at")}
+                                          if c.get("kind") == "history" and c.get("higher")
+                                          else None)})
+    return {"language": language, "allegations": rows, "report_provision_held": bool(held)}

canlex/pdi.py

@@ -0,0 +1,321 @@
+"""Ingest scoped IRCC Program Delivery Instructions (PDI) pages.
+
+The PDIs are the current-policy successor to the legacy chapter manuals.
+This module ingests the specific PDI sets that replace retired ENF chapters
+the corpus needs: the PR-card/status section (successor to ENF 27) and the
+Tran serious-criminality page (partial successor to ENF 2). Each named
+index page is fetched, its same-section sub-pages discovered, and every
+page chunked by heading -- doc_type='memorandum', like all guidance.
+
+    python -m canlex.pdi [--allow-shrink]
+"""
+import re
+import sys
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      split_lines, stored_chunks, uniquify_ids, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "pdi"
+OUT_FILE = PROCESSED_DIR / "pdi.json"
+_OBM = ("https://www.canada.ca/en/immigration-refugees-citizenship/corporate/"
+        "publications-manuals/operational-bulletins-manuals/")
+
+# Each set is one PDI category tree, crawled breadth-first from its index to
+# _MAX_DEPTH, capped at _MAX_PAGES pages (dropped pages are LOGGED, never
+# silently skipped). Scope fragments keep the crawl inside the category. The
+# selection is the border-relevant slice of the OBM tree: temporary
+# residents (visitors/eTA/TRV/TRPs), refugee protection (port-of-entry
+# claims), the cross-cutting standard requirements (includes the Tran
+# serious-criminality page), officer designation/delegation, biometrics and
+# identity, and the PR-card section (ENF 27's successor). Deliberately out:
+# citizenship, fees, office service-delivery, and the year-by-year
+# operational-bulletin archives (mostly expired instructions).
+_MAX_DEPTH = 2
+_MAX_PAGES = 160    # 60 capped out: the Temporary Residents tree alone has
+                    # ~150 in-scope pages (2026-07 crawl)
+
+SETS = [
+    {"code": "pdi-prcard",
+     "short": "PDI PR Card",
+     "name": "PDI — Permanent resident card and PR status (successor to ENF 27)",
+     "index": _OBM + "permanent-residence/card.html",
+     "scope": "/permanent-residence/card"},
+    {"code": "pdi-tr",
+     "short": "PDI Temporary Residents",
+     "name": "PDI — Temporary Resident Program (visitors, eTA, TRV, permits)",
+     "index": _OBM + "temporary-residents.html",
+     "scope": "/temporary-residents"},
+    {"code": "pdi-refugee",
+     "short": "PDI Refugee Protection",
+     "name": "PDI — Refugee Protection (claims, eligibility, port of entry)",
+     "index": _OBM + "refugee-protection.html",
+     "scope": "/refugee-protection"},
+    {"code": "pdi-standard",
+     "short": "PDI Standard Requirements",
+     "name": "PDI — Standard requirements (incl. Tran serious-criminality "
+             "assessment)",
+     "index": _OBM + "standard-requirements.html",
+     "scope": "/standard-requirements"},
+    {"code": "pdi-delegation",
+     "short": "PDI Designation-Delegation",
+     "name": "PDI — Designation of officers and delegation of authority",
+     "index": _OBM + "legislation.html",
+     "scope": "operational-bulletins-manuals/legislation"},
+    {"code": "pdi-identity",
+     "short": "PDI Biometrics-Identity",
+     "name": "PDI — Biometrics and identity management",
+     "index": _OBM + "identity-management.html",
+     "scope": "/identity-management"},
+    {"code": "pdi-pr",
+     "short": "PDI Permanent Residence",
+     "name": "PDI — Permanent residence (loss of status and residency "
+             "obligation, non-economic classes, legal references)",
+     "index": _OBM + "permanent-residence.html",
+     "scope": "/permanent-residence",
+     # the PR-card subtree is the pdi-prcard set; the economic classes
+     # (Express Entry, PNP, business) are not border work
+     "exclude": ("/permanent-residence/card", "/economic-classes")},
+    # ENF 3 s. 10.9 (2022) sends officers to "IRCC's Program Delivery
+    # Instructions (PDI) on Criminal Rehabilitation". No such page is
+    # published (checked 2026-09-12): the OBM index and the
+    # standard-requirements / temporary-residents trees link none, canada.ca
+    # search finds only the applicant pages (Guide 5312, IMM 1444, 'Deemed
+    # rehabilitation'), and .../operational-bulletins-manuals/
+    # criminal-rehabilitation.html and .../standard-requirements/
+    # criminal-rehabilitation.html return 404. The one public PDI text on
+    # rehabilitation is this fees-tree page (R309(a) / A36(3)(c), dated
+    # 2013-06-17 -- it still names the National Parole Board and ENF 2/OP 18).
+    # A single page, so no crawl scope.
+    {"code": "pdi-rehab",
+     "short": "PDI Rehabilitation",
+     "name": "PDI — Determination of rehabilitation (R309(a) and A36(3)(c))",
+     "index": _OBM + ("fees/immigration/general/"
+                      "determination-rehabilitation-r309-a36-3.html"),
+     "scope": ""},
+]
+
+# canada.ca records a page's modification date in <meta name="dcterms.modified"
+# content="YYYY-MM-DD">; the stored OBM pages carry no <time> element, so the
+# old <time>-only pattern left every PDI chunk undated. <time> stays as the
+# fallback for pages that do have one.
+_MODIFIED = re.compile(r'<time[^>]*>(\d{4}-\d{2}-\d{2})</time>')
+_META = re.compile(r"<meta\b[^>]*>", re.I)
+_META_MODIFIED = re.compile(r"""name=["']dcterms\.modified["']""", re.I)
+_META_CONTENT = re.compile(r"""content=["'](\d{4}-\d{2}-\d{2})""", re.I)
+
+
+def page_modified(html, today=None):
+    """The page's dcterms.modified date (else its <time> date), ignoring any
+    date later than `today`: the OBM translation page carries 2027-01-10,
+    which is a CMS artefact, not a date the text was current to. '' when no
+    usable date is found. Pure, for tests."""
+    from datetime import date
+    today = today or date.today().isoformat()
+    candidates = []
+    for tag in _META.findall(html):
+        if _META_MODIFIED.search(tag):
+            m = _META_CONTENT.search(tag)
+            if m:
+                candidates.append(m.group(1))
+    candidates.extend(_MODIFIED.findall(html))
+    for found in candidates:
+        if found <= today:
+            return found
+    return ""
+_TITLE = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S)
+_LINK = re.compile(r'href="([^"#?]*?\.html)"')
+
+
+def _strip_tags(html):
+    html = re.sub(r"</t[dh]>", " | ", html)
+    html = re.sub(r"</(?:p|li|tr|h\d|caption)>", "\n", html)
+    html = re.sub(r"<[^>]+>", " ", html)
+    return [l for l in (_norm(x).strip("| ").strip()
+                        for x in html.split("\n")) if l]
+
+
+def _main(html):
+    m = re.search(r"<main[^>]*>(.*?)</main>", html, re.S)
+    return m.group(1) if m else html
+
+
+def _page_chunks(src, page_url, html, page_no):
+    content = _main(html)
+    title_m = _TITLE.search(content)
+    title = (_norm(re.sub(r"<[^>]+>", " ", title_m.group(1)))
+             if title_m else src["short"])
+    modified = page_modified(html)
+    # Split on h2 headings; each becomes a section-like unit.
+    parts = re.split(r"<h2[^>]*>", content[title_m.end():] if title_m else content)
+    chunks = []
+    for i, part in enumerate(parts):
+        if i == 0:
+            heading, body_html = title, part
+        else:
+            head_end = part.find("</h2>")
+            heading = _norm(re.sub(r"<[^>]+>", " ", part[:head_end]))
+            body_html = part[head_end:]
+        body = "\n".join(_strip_tags(body_html))
+        if len(body) < 120:
+            continue
+        pieces = split_lines(body, 1800) if len(body) > 2400 else [body]
+        total = len(pieces)
+        for k, piece in enumerate(pieces, start=1):
+            suffix = "" if total == 1 else f"-p{k}"
+            note = "" if total == 1 else f" (part {k} of {total})"
+            chunks.append({
+                "id": f"{src['code']}-{page_no}-{i}{suffix}",
+                "doc_type": "memorandum",
+                "act_code": src["code"].upper(),
+                "act_short": src["short"],
+                "act_name": src["name"],
+                "section": f"{title[:60]} — {heading[:60]}" if i else title[:80],
+                "marginal_note": heading,
+                "part": src["name"],
+                "division": "",
+                "heading": "",
+                "text": piece,
+                "history": "",
+                "last_amended": modified,
+                "current_to": modified,
+                "citation": f"{src['short']}: {heading}{note}",
+                "source_url": page_url,
+            })
+    return chunks
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of sub-pages this run could not fetch.
+
+    Ingestion is a full rebuild, so a sub-page that 404s or drops its
+    connection silently vanishes from the corpus; serving the copy we already
+    had beats losing instructions that are still in force. Keyed on source_url,
+    the only per-page identity a PDI chunk carries that survives pages coming
+    and going -- the chunk id embeds the page's position in the index listing,
+    which shifts whenever one page drops out. Restricted to pages that actually
+    errored, so a sub-page IRCC retired on purpose is not resurrected.
+    Returns (chunks, preserved page URLs).
+    """
+    return preserve_dropped(chunks, stored, key=lambda c: c["source_url"],
+                            only={url for url, _why in failures})
+
+
+def cache_name(src, url):
+    """The raw-cache file name for one crawled page: the set's code plus the
+    page's path below the OBM root, so sibling pages that share a file name
+    -- the five .../non-economic-classes/<pathway>/admissibility.html pages
+    of the PR tree -- never share one cache file. Until 2026-09-21 the name
+    was the URL's last path component, and a plain (no --force) run served
+    one pathway's cached page under all five URLs. The index page keeps
+    '{code}-index.html': refresh.check_pdi reads it by that name. Pure."""
+    if url == src["index"]:
+        return f"{src['code']}-index.html"
+    rel = url.split(_OBM, 1)[1] if _OBM in url else re.sub(r"^https?://", "", url)
+    rel = re.sub(r"\.html$", "", rel)
+    return f"{src['code']}-{re.sub(r'[^A-Za-z0-9._-]+', '-', rel)}.html"
+
+
+def _crawl(src, failures, force=False):
+    """Breadth-first crawl of one PDI category: index -> in-scope links ->
+    their in-scope links, to _MAX_DEPTH, capped at _MAX_PAGES pages.
+    Returns [(url, html)]; pages dropped by the cap are LOGGED, and fetch
+    failures land in `failures` so preserve_failed keeps their stored
+    copies."""
+    index_html = fetch_cached(
+        src["index"], RAW / cache_name(src, src["index"]),
+        powershell=True, pause=0.8, force=force).decode("utf-8", "replace")
+    pages = [(src["index"], index_html)]
+    if not src["scope"]:
+        return pages
+    seen = {src["index"]}
+    frontier = [(src["index"], index_html)]
+    dropped = 0
+    for _depth in range(_MAX_DEPTH):
+        next_frontier = []
+        for _url, html in frontier:
+            for href in _LINK.findall(_main(html)):
+                if src["scope"] not in href:
+                    continue
+                if any(x in href for x in src.get("exclude", ())):
+                    continue
+                url = href if href.startswith("http") else \
+                    "https://www.canada.ca" + href
+                if url in seen:
+                    continue
+                seen.add(url)
+                if len(pages) >= _MAX_PAGES:
+                    dropped += 1
+                    continue
+                name = url.rsplit("/", 1)[1].replace(".html", "")
+                try:
+                    page = fetch_cached(
+                        url, RAW / cache_name(src, url),
+                        powershell=True, pause=0.8,
+                        force=force).decode("utf-8", "replace")
+                except Exception as exc:
+                    failures.append((url, f"{type(exc).__name__}: {exc}"))
+                    print(f"  !! {name}: {type(exc).__name__}: {exc}")
+                    continue
+                pages.append((url, page))
+                next_frontier.append((url, page))
+        frontier = next_frontier
+    if dropped:
+        print(f"  !! {src['short']}: page cap {_MAX_PAGES} hit -- {dropped} "
+              f"in-scope pages NOT ingested (raise _MAX_PAGES to cover them)")
+    return pages
+
+
+def build(allow_shrink=False, force=False):
+    """Rebuild pdi.json. Returns the chunks, or False if the write was refused."""
+    all_chunks, failures = [], []
+    for src in SETS:
+        try:
+            pages = _crawl(src, failures, force=force)
+        except Exception as exc:
+            failures.append((src["index"], f"{type(exc).__name__}: {exc}"))
+            print(f"  !! {src['short']}: index fetch failed "
+                  f"({type(exc).__name__}: {exc})")
+            continue
+        n = 0
+        for page_no, (url, html) in enumerate(pages):
+            chunks = _page_chunks(src, url, html, page_no)
+            if not chunks:
+                # Downloaded fine, parsed to nothing: the page template moved
+                # under us. That is exactly how the D-memo corpus was lost, so
+                # it counts as a failure and the stored page is kept.
+                print(f"  !! {url.rsplit('/', 1)[-1]}: no content parsed; "
+                      f"keeping stored copy")
+                failures.append((url, "no content parsed"))
+                continue
+            all_chunks.extend(chunks)
+            n += len(chunks)
+        print(f"  {src['short']}: {len(pages)} pages, {n} chunks")
+
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT_FILE))
+    if preserved:
+        names = ", ".join(u.rsplit("/", 1)[-1] for u in preserved)
+        print(f"  kept the last-good copy of {len(preserved)} unfetchable "
+              f"page(s): {names}")
+    # After preservation, not before: a preserved page brings back the ids it
+    # was written with, and this run may have handed those ids to other pages
+    # (an id embeds the page's position in the index listing).
+    uniquify_ids(all_chunks)
+    if not write_corpus(OUT_FILE, all_chunks, indent=2,
+                        allow_shrink=allow_shrink):
+        return False
+    print(f"pdi: {len(all_chunks)} chunks -> {OUT_FILE.name}")
+    return all_chunks
+
+
+def main():
+    # `is not False` rather than truthiness: a first-ever run that writes an
+    # empty corpus succeeded, a refused write did not.
+    ok = build(allow_shrink="--allow-shrink" in sys.argv,
+               force="--force" in sys.argv) is not False
+    sys.exit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/ratelimit.py

@@ -0,0 +1,238 @@
+"""Per-client limits for the HTTP transport (audit follow-up 2026-09-21,
+missing 14; verification fixes 2026-09-22, SEC-1 and SEC-2).
+
+The public Space is unauthenticated. The request-body cap, the session idle
+timeout and the off-loop tools bound what one request can cost, but a steady
+stream of 500-character searches (2.5-7 s each, one at a time) still holds the
+single search slot, and a stream of `initialize` requests opens sessions that
+each live until the idle timeout. Two small ASGI middlewares, wrapped around
+_Utf8Body by server.http_app, give each client a token bucket:
+
+- SessionOpenLimit (ON by default over HTTP): a POST with no mcp-session-id
+  header is the only request that can open an MCP session; each client may
+  send SESSION_OPEN_LIMIT (60) of them per SESSION_OPEN_WINDOW (10 minutes),
+  in bursts of up to 60, and then one each 10 s. CANLEX_SESSION_OPEN_LIMIT
+  sets the number (0 turns the cap off). An officer's client opens one
+  session per connection, so it never meets the cap; one client can no
+  longer fill the session table (SEC-1: mcp 1.30.0's global cap of 10,000
+  let a single client lock every other one out with 503 for 30 minutes;
+  server.py now builds the manager with no global cap).
+- RateLimit (OFF unless CANLEX_RATE_LIMIT_PER_MIN is set to a positive
+  integer): `per_minute` requests a minute on average and bursts of up to
+  `burst` (CANLEX_RATE_LIMIT_BURST; default a full minute's allowance, so an
+  officer's own client -- a few requests per tool call -- never meets it).
+
+A request over either limit gets 429 with a plain-text body and Retry-After;
+it never reaches the MCP app. GET / (the notice page and health probes) and
+anything that is not an HTTP request (lifespan) pass through uncounted. The
+local stdio server uses neither.
+
+The client (client_key) is the address the last proxy saw: the rightmost
+X-Forwarded-For entry when the header is present -- repeated header lines
+read as one comma-separated list, in order -- else the socket peer, cut to
+MAX_KEY_CHARS characters. This ASSUMES AN APPENDING PROXY in front: Hugging
+Face's ingress appends the address it saw (and merges any lines the client
+sent), so the rightmost entry is not the client's to choose. Served with no
+such proxy (plain Docker, or a proxy that passes the header through
+untouched), the header is whatever the client writes: it can dodge its own
+limit by rotating it, or spend another address's bucket by naming it. At
+most MAX_CLIENTS buckets are kept per limit; idle ones are dropped first.
+"""
+import math
+import os
+import sys
+import threading
+import time
+
+ENV_PER_MIN = "CANLEX_RATE_LIMIT_PER_MIN"
+ENV_BURST = "CANLEX_RATE_LIMIT_BURST"
+ENV_SESSIONS = "CANLEX_SESSION_OPEN_LIMIT"
+MAX_CLIENTS = 10000
+MAX_KEY_CHARS = 64          # an IPv6 address with a zone is under 64 characters
+SESSION_OPEN_LIMIT = 60     # new sessions per client per SESSION_OPEN_WINDOW
+SESSION_OPEN_WINDOW = 600.0  # seconds
+
+
+class TokenBuckets:
+    """One token bucket per client key: `rate` tokens a second, at most
+    `burst` held. take(key) spends one token and returns 0.0, or returns the
+    seconds until one is available (nothing spent)."""
+
+    def __init__(self, per_minute, burst=None, clock=time.monotonic,
+                 max_clients=MAX_CLIENTS):
+        self.rate = per_minute / 60.0
+        self.burst = float(burst if burst else per_minute)
+        self.clock = clock
+        self.max_clients = max_clients
+        self._buckets = {}          # key -> (tokens, last refill time)
+        self._lock = threading.Lock()
+
+    def __len__(self):
+        return len(self._buckets)
+
+    def take(self, key):
+        with self._lock:
+            now = self.clock()
+            tokens, last = self._buckets.get(key, (self.burst, now))
+            tokens = min(self.burst, tokens + (now - last) * self.rate)
+            if tokens >= 1.0:
+                self._buckets[key] = (tokens - 1.0, now)
+                self._prune(now)
+                return 0.0
+            self._buckets[key] = (tokens, now)
+            return (1.0 - tokens) / self.rate
+
+    def _prune(self, now):
+        """Keep at most max_clients buckets: drop the full ones (idle long
+        enough to refill), then the least recently seen."""
+        if len(self._buckets) <= self.max_clients:
+            return
+        full = [k for k, (t, last) in self._buckets.items()
+                if t + (now - last) * self.rate >= self.burst]
+        for k in full:
+            del self._buckets[k]
+        if len(self._buckets) > self.max_clients:
+            by_age = sorted(self._buckets, key=lambda k: self._buckets[k][1])
+            for k in by_age[:len(self._buckets) - self.max_clients]:
+                del self._buckets[k]
+
+
+def client_key(scope):
+    """The address the last proxy saw: the rightmost X-Forwarded-For entry
+    (every header line joined in order, so a proxy that adds its own line is
+    read, not the client's first one), else the socket peer's host, else
+    '?'; at most MAX_KEY_CHARS characters. Trusts the header from any peer:
+    it assumes an appending proxy in front (see the module docstring)."""
+    lines = [value.decode("latin-1") for name, value in scope.get("headers") or ()
+             if name.lower() == b"x-forwarded-for"]
+    if lines:
+        last = ",".join(lines).split(",")[-1].strip()
+        if last:
+            return last[:MAX_KEY_CHARS]
+    client = scope.get("client")
+    return str(client[0])[:MAX_KEY_CHARS] if client else "?"
+
+
+def _exempt(scope):
+    return scope.get("type") != "http" or (scope.get("method") == "GET"
+                                           and scope.get("path") == "/")
+
+
+def _opens_session(scope):
+    """A POST with no mcp-session-id header: the only request the session
+    manager answers by opening a session (a request naming an unknown id gets
+    404 and opens nothing)."""
+    return (scope.get("type") == "http" and scope.get("method") == "POST"
+            and not any(name.lower() == b"mcp-session-id"
+                        for name, _ in scope.get("headers") or ()))
+
+
+async def _too_many(send, wait, text):
+    """429 with `text` and the whole seconds to wait, as a plain-text body
+    and Retry-After."""
+    retry = max(1, math.ceil(wait))
+    body = f"{text} Retry after {retry} s.\n".encode("utf-8")
+    await send({"type": "http.response.start", "status": 429,
+                "headers": [(b"content-type", b"text/plain; charset=utf-8"),
+                            (b"content-length", str(len(body)).encode("ascii")),
+                            (b"retry-after", str(retry).encode("ascii"))]})
+    await send({"type": "http.response.body", "body": body})
+
+
+class RateLimit:
+    """ASGI middleware: each client's requests through a TokenBuckets."""
+
+    def __init__(self, app, per_minute, burst=None, clock=time.monotonic,
+                 max_clients=MAX_CLIENTS):
+        self.app = app
+        self.per_minute = per_minute
+        self.buckets = TokenBuckets(per_minute, burst, clock, max_clients)
+
+    async def __call__(self, scope, receive, send):
+        if _exempt(scope):
+            await self.app(scope, receive, send)
+            return
+        wait = self.buckets.take(client_key(scope))
+        if not wait:
+            await self.app(scope, receive, send)
+            return
+        await _too_many(send, wait, f"Too many requests: this CanLex endpoint allows "
+                                    f"{self.per_minute} requests a minute per client.")
+
+
+class SessionOpenLimit:
+    """ASGI middleware: each client may send `limit` session-opening
+    requests (_opens_session) per `window` seconds, in bursts of up to
+    `limit`; every other request passes through uncounted."""
+
+    def __init__(self, app, limit=SESSION_OPEN_LIMIT, window=SESSION_OPEN_WINDOW,
+                 clock=time.monotonic, max_clients=MAX_CLIENTS):
+        self.app = app
+        self.limit, self.window = limit, window
+        self.buckets = TokenBuckets(limit * 60.0 / window, limit, clock, max_clients)
+
+    async def __call__(self, scope, receive, send):
+        if not _opens_session(scope):
+            await self.app(scope, receive, send)
+            return
+        wait = self.buckets.take(client_key(scope))
+        if not wait:
+            await self.app(scope, receive, send)
+            return
+        await _too_many(send, wait, f"Too many new sessions: this CanLex endpoint lets one "
+                                    f"client open {self.limit} MCP sessions every "
+                                    f"{self.window / 60:g} minutes.")
+
+
+def _positive_int(environ, name):
+    raw = (environ.get(name) or "").strip()
+    if not raw:
+        return None
+    try:
+        value = int(raw)
+    except ValueError:
+        value = 0
+    if value <= 0:
+        print(f"CanLex MCP: {name}={raw!r} is not a positive integer; ignored",
+              file=sys.stderr)
+        return None
+    return value
+
+
+def from_env(app, environ=None):
+    """`app` wrapped in RateLimit when CANLEX_RATE_LIMIT_PER_MIN is a positive
+    integer (one stderr line says so), else `app` itself."""
+    environ = os.environ if environ is None else environ
+    per_minute = _positive_int(environ, ENV_PER_MIN)
+    if per_minute is None:
+        return app
+    burst = _positive_int(environ, ENV_BURST) or per_minute
+    print(f"CanLex MCP: rate limit {per_minute} requests a minute per client "
+          f"(bursts of {burst}); GET / exempt.", file=sys.stderr)
+    return RateLimit(app, per_minute, burst)
+
+
+def sessions_from_env(app, environ=None):
+    """`app` wrapped in SessionOpenLimit -- on by default: SESSION_OPEN_LIMIT
+    new sessions per client per SESSION_OPEN_WINDOW; CANLEX_SESSION_OPEN_LIMIT
+    sets the number, and 0 turns the cap off (`app` itself). A value that is
+    not a whole number keeps the default. One stderr line says which."""
+    environ = os.environ if environ is None else environ
+    raw = (environ.get(ENV_SESSIONS) or "").strip()
+    limit = SESSION_OPEN_LIMIT
+    if raw:
+        try:
+            limit = int(raw)
+        except ValueError:
+            limit = -1
+        if limit == 0:
+            print(f"CanLex MCP: no per-client cap on new sessions ({ENV_SESSIONS}=0).",
+                  file=sys.stderr)
+            return app
+        if limit < 0:
+            print(f"CanLex MCP: {ENV_SESSIONS}={raw!r} is not a whole number; the default "
+                  f"{SESSION_OPEN_LIMIT} applies", file=sys.stderr)
+            limit = SESSION_OPEN_LIMIT
+    print(f"CanLex MCP: at most {limit} new sessions per client every "
+          f"{SESSION_OPEN_WINDOW / 60:g} minutes.", file=sys.stderr)
+    return SessionOpenLimit(app, limit)

canlex/readnext.py

@@ -0,0 +1,224 @@
+"""'Read next': the official guidance and statute a curated entry names (roadmap T3.6).
+
+A curated disposition entry, methodology item, equivalency pairing or transborder
+row may carry `see_also` (HELD FOR REVIEW when added; optional -- the approved
+baseline predates it): an ordered list of items, each a STABLE key --
+'<act_code>|s. <section>' for a statute section, '<act_code>|<ENF section>' for a
+numbered ENF section, '<act_code>|<marginal note>' for an IRB paper, PDI page or
+IRCC guide section, '<act_code>|ENF 1 App. A|<case name>' for an ENF 1 App. A
+note -- with the chunk it quotes ('chunk') and, optionally, a verbatim passage
+of that chunk ('quote', at most 150 characters; tests/test_read_next.py pins
+each one). This is the shape canlex/index.py's loader reads for co-surfacing
+(curated_see_also, _see_also_spec, _see_also_match); the rule that resolves a
+key is that module's, imported here, so the tools and the search index read a
+key alike.
+
+The tools print, for the entry the record matched, one line per item: the
+corpus's own label for the chunk (act, section, marginal note), its chunk id,
+the source's status (an unofficial ATIP copy of a withdrawn chapter is flagged;
+guidance shows its date) and the quote. No prose of CanLex's own is printed:
+the 'why' is the source's own words. Official sources come first; an ATIP copy
+follows them. A key the loaded corpus cannot resolve is printed as such, never
+dropped silently.
+"""
+import functools
+import json
+import re
+
+from .config import PROCESSED_DIR
+from .index import ATIP_STATUS, _see_also_match, _see_also_spec
+
+HEADING = ("READ NEXT -- the official guidance and statute on this point, from the corpus "
+           "(chunk id in brackets):")
+COMPACT_PREFIX = "Read next"
+ATIP_FLAG = "UNOFFICIAL ATIP COPY of a withdrawn chapter -- historical guidance"
+ATIP_SHORT = "unofficial ATIP copy"
+UNRESOLVED = "not in this deployment's corpus"
+
+
+def items(obj):
+    """The see_also items of a curated object ([] when it has none)."""
+    raw = obj.get("see_also") if isinstance(obj, dict) else None
+    return [i for i in raw or [] if isinstance(i, (dict, str))]
+
+
+def _file_for_act(act_code):
+    """The data/processed file holding an act_code's chunks."""
+    code = act_code or ""
+    if code.startswith("ENF-"):
+        return "enf.json"
+    if code.startswith("IRB-LS-"):
+        return "irb_papers.json"
+    if code.startswith("IRB-"):
+        return "irb_guidelines.json"
+    if code.startswith("PDI-"):
+        return "pdi.json"
+    if code.startswith(("IMM-", "IRCC-")):
+        return "policy.json"
+    return f"{code}.json"
+
+
+@functools.lru_cache(maxsize=48)
+def _file_chunks(name):
+    """{act_code: [English chunk, ...]} of one processed file ({} when absent)."""
+    try:
+        data = json.loads((PROCESSED_DIR / name).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+    if isinstance(data, dict):
+        data = data.get("chunks") or []
+    out = {}
+    for c in data:
+        if isinstance(c, dict) and c.get("lang", "en") != "fr":
+            out.setdefault(c.get("act_code"), []).append(c)
+    return out
+
+
+_INDEX_MAP = []      # [(the loaded index's chunk list, {act_code: [chunk, ...]})]
+
+
+def _pool(act_code, index_chunks=None):
+    """The English chunks of one act_code: from the loaded index when there is
+    one (it holds the whole corpus), else from data/processed."""
+    if index_chunks is not None:
+        # read once into a local and publish whole (audit follow-up 2026-09-21):
+        # a second screening thread never reads a map another thread replaced
+        entry = _INDEX_MAP[0] if _INDEX_MAP else None
+        if entry is None or entry[0] is not index_chunks:
+            by_act = {}
+            for c in index_chunks:
+                if c.get("lang", "en") != "fr":
+                    by_act.setdefault(c.get("act_code"), []).append(c)
+            entry = (index_chunks, by_act)
+            _INDEX_MAP[:] = [entry]
+        got = entry[1].get(act_code)
+        if got:
+            return got
+    return _file_chunks(_file_for_act(act_code)).get(act_code, [])
+
+
+def resolve(item, index_chunks=None):
+    """{key, id, chunk, quote}: the chunk an item names -- its pinned chunk when
+    that still fits the key, else the key's first English piece (the rule
+    canlex/index.py co-surfaces by); chunk None when nothing resolves. The
+    quote is kept only with the chunk it was taken from."""
+    spec, pin = _see_also_spec(item, {})
+    key = item.get("key") if isinstance(item, dict) else item
+    quote = item.get("quote") if isinstance(item, dict) else None
+    chunk = None
+    if spec:
+        pool = _pool(spec.get("act_code"), index_chunks)
+        if pin:
+            chunk = next((c for c in pool if c.get("id") == pin), None)
+            if chunk is not None and not _see_also_match(chunk, spec):
+                chunk = None
+        if chunk is None:
+            chunk = next((c for c in pool if _see_also_match(c, spec)), None)
+    return {"key": key, "id": chunk.get("id") if chunk else None, "chunk": chunk,
+            "quote": quote if chunk is not None and chunk.get("id") == pin else None}
+
+
+def is_atip(chunk):
+    return bool(chunk and chunk.get("source_status") == ATIP_STATUS)
+
+
+def is_guidance(chunk):
+    return bool(chunk and chunk.get("doc_type") == "memorandum")
+
+
+def resolved(item_list, index_chunks=None):
+    """The items resolved, one per chunk (the first naming it), official
+    sources first and unofficial ATIP copies after them, each group in curated
+    order; an unresolved item keeps its place in the official group."""
+    out, seen = [], set()
+    for it in item_list or ():
+        r = resolve(it, index_chunks)
+        if r["id"] is not None:
+            if r["id"] in seen:
+                continue
+            seen.add(r["id"])
+        out.append(r)
+    return [r for r in out if not is_atip(r["chunk"])] + [r for r in out if is_atip(r["chunk"])]
+
+
+def label(chunk):
+    """The corpus's own name for a chunk: 'Criminal Code s. 320.14 (Operation
+    while impaired)', 'ENF 3 s. 10.9 (Foreign pardons)', 'IRB Legal Services,
+    ROA ch. 8 § 18 (Effect of a Pardon)', 'PDI Standard Requirements (Where the
+    ruling applies)'."""
+    short = chunk.get("act_short") or ""
+    sec = chunk.get("section") or ""
+    note = chunk.get("marginal_note") or ""
+    if re.match(r"\d", sec):
+        base = f"{short} s. {sec}"
+    elif short and sec.startswith(short):
+        base = sec
+    elif short.startswith("IRB Legal Services") and sec:
+        base = f"IRB Legal Services, {sec}"
+    else:
+        base = short or sec
+    if note and note.lower() not in base.lower():
+        base += f" ({note})"
+    return base
+
+
+def flags(chunk):
+    """The source's status: the ATIP flag, an archived paper, a guidance date."""
+    out = []
+    if is_atip(chunk):
+        out.append(ATIP_FLAG)
+    if "archived" in (chunk.get("act_name") or "").lower():
+        out.append("archived")
+    if is_guidance(chunk) and chunk.get("current_to"):
+        out.append(f"as of {chunk['current_to']}")
+    return out
+
+
+def lines(item_list, index_chunks=None, heading=HEADING):
+    """The READ NEXT block: the heading and one line per item ([] when the
+    entry names none)."""
+    res = resolved(item_list, index_chunks)
+    if not res:
+        return []
+    out = [heading]
+    for r in res:
+        if r["chunk"] is None:
+            out.append(f"- {r['key']}: {UNRESOLVED}")
+            continue
+        fl = flags(r["chunk"])
+        text = f"- {label(r['chunk'])} [{r['id']}]" + (f" -- {'; '.join(fl)}" if fl else "")
+        if r["quote"]:
+            text += f': "{r["quote"]}"'
+        out.append(text)
+    return out
+
+
+def compact_line(item_list, index_chunks=None, prefix=COMPACT_PREFIX):
+    """One line for a memo: 'Read next: <label> [<id>]; ...' (ATIP copies
+    marked), or None when the entry names nothing."""
+    res = resolved(item_list, index_chunks)
+    if not res:
+        return None
+    parts = []
+    for r in res:
+        if r["chunk"] is None:
+            parts.append(f"{r['key']} ({UNRESOLVED})")
+        else:
+            parts.append(f"{label(r['chunk'])} [{r['id']}]"
+                         + (f" ({ATIP_SHORT})" if is_atip(r["chunk"]) else ""))
+    return f"{prefix}: " + "; ".join(parts)
+
+
+def chunk_ids(item_list, index_chunks=None):
+    """The resolved chunk ids, in printed order (CANLEX_RESULT 'read_next')."""
+    return [r["id"] for r in resolved(item_list, index_chunks) if r["id"]]
+
+
+def with_fallback(primary, fallback, index_chunks=None):
+    """A pairing's items, followed by the equivalency framework's when none of
+    the pairing's own resolves to guidance (the draft's rule: most pairings
+    name only their Canadian statute sections)."""
+    primary = list(primary or [])
+    if not primary or any(is_guidance(r["chunk"]) for r in resolved(primary, index_chunks)):
+        return primary
+    return primary + list(fallback or [])

canlex/refresh.py

@@ -0,0 +1,1653 @@
+"""Detect when CanLex's ingested sources have gone stale upstream.
+
+    py -m canlex.refresh --check
+
+For every Justice Laws instrument in config.SOURCES, this fetches just the
+opening of the XML (the root <Statute>/<Regulation> element carries the
+consolidation date in its lims:current-date attribute, so a few KB is enough)
+and compares that upstream date against the current_to stored in the matching
+data/processed/<code>.json. Anything where upstream is newer is reported as
+drift, and the process exits non-zero -- so this can gate a deploy or run on a
+schedule.
+
+It also checks the two delegation instruments against upstream (check_delegation):
+the CBSA IRPA delegation -- by diffing the dated 'irpa-lipr-YYYY-MM-DD' instruments
+the CBSA legislation index publishes against those delegation.SOURCES ingests, so
+a newly-published amendment is flagged -- and the IRCC IL3 instrument, by comparing
+the version stamp in the live PDF against the stored one. The NJC directives get
+the same treatment (check_directives): the effective date each chunk stores in
+current_to comes from the NJC directive index (the span.date beside each entry),
+so one fresh index fetch re-reads it for nearly every directive, with the
+'effective <Month D, YYYY>' prose on the directive's own page as fallback for
+entries the index leaves undated (the Foreign Service Directives) -- exactly the
+two signals ingestion uses. Finally it prints the stored currency of the
+remaining non-XML sources (D-memos, case law, agreements, benefits, tariff
+schedule), which expose no comparable upstream signal, with the command to
+refresh each; and flags any tracked pending bill whose coming-into-force date
+has passed, saying whether the stored consolidation of each affected Act
+already postdates it (pending_consolidation).
+
+Two further watches, both added 2026-09-13:
+
+- Curated verify phrases (check_curated_sections, roadmap T4.3). Every statute
+  phrase the curated tables pin is checked against the corpus now. BROKEN
+  means the phrase or section is gone. AMENDED means the section's
+  last_amended is later than in the corpus of the approved commit; when the
+  maximum a quoted subsection states changed and no penalty_history row
+  covers it, a DRAFT row goes to data/curated/drafts/. The curated JSON
+  itself is never edited here (see canlex.curated_checks).
+- Guidance currency (check_guidance, roadmap T4.8):
+  - the official ENF chapters, by an HTTP HEAD of each PDF against the
+    last-seen marker (ENF 2 and 14, withdrawn, are skipped);
+  - each PDI category index, diffed against the index cached at ingest,
+    plus the Tran page;
+  - the IRB legal-concepts index pages, for their ARCHIVED marker.
+  IMM 1444 and Guide 5312 are read from the employer-policy check above,
+  which already stamps their page dateModified.
+
+This is the staleness detection the corpus previously lacked: legislation was
+refreshed by hand, so e.g. Bill C-12 only landed when someone thought to look.
+
+    py -m canlex.refresh --check          # human-readable; exit 1 on drift
+    py -m canlex.refresh --check --json    # machine-readable for CI
+"""
+import datetime
+import json
+import os
+import re
+import sys
+import tempfile
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+
+from . import delegation, directive, dmemo
+from ._common import BROWSER_UA, MONTHS, atomic_write_text
+from .config import SOURCES, PROCESSED_DIR
+
+_UA = "CanLex-refresh/0.1"
+_HEAD_BYTES = 16384      # the root element's attributes sit at the very top
+_CURRENT_DATE = re.compile(rb'current-date="(\d{4}-\d{2}-\d{2})"')
+
+# Aggregate (non-Justice-Laws) sources: (label, processed filename, refresh
+# command, description). These carry no single upstream consolidation date, so
+# the check reports their STORED currency (newest item) and how to refresh them
+# rather than diffing against upstream. (Delegation and the NJC directives are
+# checked against upstream separately, by check_delegation and check_directives,
+# because their sources DO expose a comparable signal -- the CBSA instrument
+# index and IL3 version string, and the NJC directive index dates.)
+_NON_XML = [
+    ("memorandum", "dmemos.json", "py -m canlex.dmemo", "CBSA D-Memoranda"),
+    ("caselaw", "caselaw.json", "py -m canlex.caselaw", "Court & tribunal decisions"),
+    ("agreement", "agreements.json", "py -m canlex.agreement", "FB collective agreement"),
+    ("benefits", "benefits.json", "py -m canlex.benefits", "Benefit-plan booklets"),
+    ("tariff", "tariff_schedule.json", "py -m canlex.tariff_schedule",
+     "Customs Tariff Schedule ch. 98-99"),
+    ("enf", "enf.json", "py -m canlex.enf", "ENF operational manuals"),
+    ("amps", "amps.json", "py -m canlex.amps", "AMPS Master Penalty Document"),
+    ("charter", "charter.json", "py -m canlex.charter", "Charter / Constitution"),
+    ("commentary", "commentary.json", "py -m canlex.commentary",
+     "Curated commentary chunks"),
+    ("union", "ciu.json", "py -m canlex.ciu", "CIU/PSAC union documents"),
+    ("pdi", "pdi.json", "py -m canlex.pdi",
+     "IRCC PDI trees (TR/refugee/standard/identity/PR-card)"),
+    ("memorandum-fr", "dmemos-fr.json", "py -m canlex.dmemo --fr",
+     "Mémorandums D (français)"),
+    ("directive-fr", "directives-fr.json", "py -m canlex.directive --fr",
+     "Directives du CNM (français)"),
+    ("irb-guidelines", "irb_guidelines.json", "py -m canlex.irb_guidelines",
+     "IRB Chairperson's Guidelines"),
+    ("irb-papers", "irb_papers.json", "py -m canlex.irb_papers",
+     "IRB Legal Services papers (ROA ch. 7-9, SPO ch. 2; archived by the IRB)"),
+    ("policy", "policy.json", "py -m canlex.conduct",
+     "Policy pages (employer instruments EN/FR, CBSA detention policy, STCA, "
+     "IRCC rehabilitation pages)"),
+    ("agreement-fr", "agreements-fr.json", "py -m canlex.agreement --fr",
+     "Convention collective FB (français)"),
+    ("enf-fr", "enf-fr.json", "py -m canlex.enf --fr",
+     "Guides ENF (édition française)"),
+]
+
+
+def _fetch_powershell(url, timeout=180):
+    """The .NET HTTP stack, which canada.ca accepts while blocking Python's
+    client at the TLS layer (the same helper _common.fetch_cached uses, so
+    the two quote a URL the same way)."""
+    from ._common import powershell_download
+    with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as tmp:
+        dest = tmp.name
+    try:
+        powershell_download(url, dest, ua=BROWSER_UA, timeout=timeout)
+        with open(dest, "rb") as fh:
+            return fh.read().decode("utf-8", "replace")
+    finally:
+        try:
+            os.unlink(dest)
+        except OSError:
+            pass
+
+
+def check_policy():
+    """Upstream dateModified of each employer-policy page vs the stored
+    currency -- both canada.ca-family pages stamp one, and the CBSA Code is
+    re-issued in place at the same URL (2024 edition, 2012 predecessor)."""
+    from .conduct import DOCS, _MODIFIED
+    # Keyed by (code, lang): the ten French twins share their English page's
+    # act_code, and a French page re-issued on its own date (the CBSA Code and
+    # the TBS instruments are re-issued per language) must be compared with
+    # ITS stored date, not the English one's -- until 2026-09-21 the FR row
+    # read STALE that --force could never clear, or the reverse silently.
+    stored_by_code = {}
+    path = PROCESSED_DIR / "policy.json"
+    if path.exists():
+        for c in json.loads(path.read_text(encoding="utf-8")):
+            stored_by_code.setdefault((c["act_code"], c.get("lang", "en")),
+                                      c.get("current_to", ""))
+    rows = []
+    for doc in DOCS:
+        stored = stored_by_code.get((doc["code"], doc.get("lang", "en")), "")
+        # Fetch the way the ingester does: a document with several pages
+        # ('urls') is stamped on its first page, and the canada.ca pages that
+        # refuse Python's HTTP client need the PowerShell path -- without
+        # this, the detention standards, the STCA texts and the rehabilitation
+        # guide could never clear, and three permanent ERROR rows teach the
+        # reader to ignore the watch.
+        url = doc.get("url") or doc["urls"][0]
+        try:
+            if doc.get("powershell"):
+                html = _fetch_powershell(url)
+            else:
+                req = urllib.request.Request(
+                    url, headers={"User-Agent": BROWSER_UA})
+                with urllib.request.urlopen(req, timeout=60) as resp:
+                    html = resp.read().decode("utf-8", "replace")
+            m = _MODIFIED.search(html)
+            remote = ((m.group(1) or m.group(2)) if m else "")
+        except Exception as exc:
+            rows.append({"short": doc["short"], "lang": doc.get("lang", "en"),
+                         "stored": stored, "remote": "", "status": "error",
+                         "error": f"{type(exc).__name__}: {exc}"})
+            continue
+        if not stored:
+            status = "missing-local"
+        elif not remote:
+            status = "no-date"
+        elif remote > stored:
+            status = "stale"
+        else:
+            status = "ok"
+        # lang: ten documents have a French twin under the same short label,
+        # so the summary tags the twin '(fr)' instead of printing each twice.
+        rows.append({"short": doc["short"], "lang": doc.get("lang", "en"),
+                     "stored": stored, "remote": remote, "status": status, "error": ""})
+        time.sleep(0.5)
+    return rows
+
+
+# ---- guidance currency: ENF, PDI, IRB papers, forms (roadmap T4.8) -----------
+# The non-XML guidance sources expose different upstream signals, so each is
+# read the cheapest way it offers:
+# - official ENF chapters (canada.ca PDFs): the HTTP Last-Modified of a HEAD
+#   request, against the last-seen marker -- the cached PDF's fetch time, or a
+#   date acknowledged with --ack-guidance, whichever is later. A 404 on a
+#   chapter the corpus holds means IRCC retired it. ENF 2 and 14 are skipped:
+#   withdrawn, and ingested from third-party ATIP copies (enf.UNOFFICIAL).
+# - PDI: each category's index page only, not all ~300 pages. The in-scope
+#   links it lists are diffed against the index cached at the last ingest, and
+#   its dcterms.modified is compared; plus the Tran serious-criminality page.
+# - IRB Legal Services papers: the ARCHIVED marker and the date modified on
+#   the legal-concepts index pages (RoaAmr.aspx, SpoPar.aspx).
+# - IMM 1444 and Guide 5312: canlex.conduct ingests both, so check_policy
+#   already reads their page dateModified; check_guidance reports those rows
+#   and does not fetch them again.
+_GUIDANCE_MARKER = PROCESSED_DIR.parent / "raw" / "guidance" / "last-seen.json"
+_GUIDANCE_FORMS = ("IMM-1444", "IMM-5312")
+_PDI_KEY_PAGES = [("pdi-standard", "PDI Tran serious-criminality page",
+                   "standard-requirements/tran.html")]
+_GUIDANCE_FLAGS = {"enf": ("check", "retired"),
+                   "pdi": ("changed", "updated", "retired"),
+                   "irb": ("updated", "unarchived")}
+
+# One PowerShell process for a batch of URLs (canada.ca refuses Python's TLS
+# client; one process instead of one per URL keeps ~50 requests to a minute
+# or two). Passed with -EncodedCommand, so no quoting survives the Windows
+# command line to mangle it.
+_PS_PROBE = r"""
+$ProgressPreference = 'SilentlyContinue'
+$urls = @(Get-Content -LiteralPath '{urls}' -Encoding UTF8 | Where-Object {{ $_ }})
+$rows = @()
+$i = 0
+foreach ($u in $urls) {{
+  $i++
+  # [string]: a Get-Content line carries PSPath/ReadCount note properties
+  # that ConvertTo-Json would serialise as an object
+  $row = [ordered]@{{url = [string]$u; status = 0; last_modified = ''; path = ''; error = ''}}
+  try {{
+    if ('{method}' -eq 'HEAD') {{
+      $r = Invoke-WebRequest -Uri $u -Method Head -UseBasicParsing -TimeoutSec 60 -UserAgent '{ua}'
+    }} else {{
+      $p = Join-Path '{out}' ("$i.html")
+      $r = Invoke-WebRequest -Uri $u -UseBasicParsing -TimeoutSec 90 -UserAgent '{ua}' -OutFile $p -PassThru
+      $row.path = [string]$p
+    }}
+    $row.status = [int]$r.StatusCode
+    $row.last_modified = [string]$r.Headers['Last-Modified']
+  }} catch {{
+    if ($_.Exception.Response) {{ $row.status = [int]$_.Exception.Response.StatusCode }}
+    $row.error = [string]$_.Exception.Message
+  }}
+  $rows += [pscustomobject]$row
+  Start-Sleep -Milliseconds {pause}
+}}
+ConvertTo-Json -InputObject $rows -Depth 3 | Set-Content -LiteralPath '{result}' -Encoding UTF8
+"""
+
+
+def _ps_probe_script(urls_file, method, out_dir, result_file, pause_ms=300):
+    return _PS_PROBE.format(urls=urls_file, method=method, out=out_dir,
+                            ua=BROWSER_UA, pause=pause_ms, result=result_file)
+
+
+def _powershell_probe(urls, method="HEAD", pause_ms=300, timeout=1800):
+    """{url: {status, last_modified, html, error}} for every URL, fetched by a
+    single PowerShell process. HEAD returns headers only; GET also returns
+    the body. status is 0 when no HTTP status came back."""
+    import base64
+    import subprocess
+    urls = list(dict.fromkeys(urls))
+    if not urls:
+        return {}
+    with tempfile.TemporaryDirectory() as td:
+        td = Path(td)
+        (td / "urls.txt").write_text("\n".join(urls), encoding="utf-8")
+        script = _ps_probe_script(td / "urls.txt", method, td, td / "result.json",
+                                  pause_ms)
+        encoded = base64.b64encode(script.encode("utf-16-le")).decode("ascii")
+        try:
+            subprocess.run(["powershell", "-NoProfile", "-NonInteractive",
+                            "-EncodedCommand", encoded],
+                           check=True, capture_output=True, timeout=timeout)
+            rows = json.loads((td / "result.json").read_text(encoding="utf-8-sig"))
+        except Exception as exc:
+            err = f"{type(exc).__name__}: {exc}"
+            return {u: {"status": 0, "last_modified": "", "html": "", "error": err}
+                    for u in urls}
+        out = {}
+        for r in rows if isinstance(rows, list) else [rows]:
+            path = Path(r["path"]) if r.get("path") else None
+            html = (path.read_bytes().decode("utf-8", "replace")
+                    if path and path.exists() else "")
+            out[r["url"]] = {"status": int(r.get("status") or 0),
+                             "last_modified": r.get("last_modified") or "",
+                             "html": html, "error": r.get("error") or ""}
+        return out
+
+
+def _guidance_marker(path=None):
+    try:
+        return json.loads(Path(path or _GUIDANCE_MARKER).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _write_guidance_marker(marker, path=None):
+    # Atomic (2026-09-22): a run killed mid-write left truncated JSON, which
+    # _guidance_marker reads as {} -- every acknowledged date silently gone
+    # and the ENF baseline re-seeded from the cached PDFs' fetch times.
+    atomic_write_text(Path(path or _GUIDANCE_MARKER),
+                      json.dumps(marker, indent=1, sort_keys=True) + "\n")
+
+
+def _http_ts(value):
+    """POSIX time of an HTTP date ('Mon, 17 Feb 2025 16:38:16 GMT'), else None."""
+    import email.utils
+    try:
+        return email.utils.parsedate_to_datetime(value).timestamp()
+    except (TypeError, ValueError, IndexError):
+        return None
+
+
+def _iso_ts(ts):
+    return (datetime.datetime.fromtimestamp(ts, datetime.timezone.utc)
+            .strftime("%Y-%m-%d %H:%M UTC") if ts is not None else "")
+
+
+def enf_targets(processed_dir=None):
+    """The official ENF chapters the corpus holds, one row per URL:
+    {chapter, lang, url, cache}. Only canada.ca editions; ENF 2 and 14 are
+    skipped (enf.UNOFFICIAL: withdrawn, known, no upstream to watch)."""
+    from . import enf
+    processed_dir = Path(processed_dir) if processed_dir else PROCESSED_DIR
+    out = []
+    for fname, lang in (("enf.json", "en"), ("enf-fr.json", "fr")):
+        path = processed_dir / fname
+        if not path.exists():
+            continue
+        seen = set()
+        for c in json.loads(path.read_text(encoding="utf-8")):
+            m = re.fullmatch(r"ENF-(\d+)", c.get("act_code", ""))
+            url = c.get("source_url", "")
+            if not m or url in seen or "canada.ca" not in url:
+                continue
+            n = int(m.group(1))
+            if n in enf.UNOFFICIAL:
+                continue
+            seen.add(url)
+            out.append({"chapter": n, "lang": lang, "url": url,
+                        "cache": enf.RAW / (f"enf{n:02d}-fra.pdf" if lang == "fr"
+                                            else f"enf{n:02d}.pdf")})
+    return sorted(out, key=lambda t: (t["lang"], t["chapter"]))
+
+
+def enf_status(probe, seen_ts):
+    """One ENF chapter's watch status from its HEAD probe. Pure. The values:
+    'retired' (a 404 on a chapter the corpus holds), 'error', 'no-date',
+    'baseline' (nothing seen yet), 'check' (Last-Modified later than last
+    seen), or 'ok'."""
+    if probe.get("status") == 404:
+        return "retired"
+    if probe.get("status") != 200:
+        return "error"
+    lm = _http_ts(probe.get("last_modified"))
+    if lm is None:
+        return "no-date"
+    if seen_ts is None:
+        return "baseline"
+    return "check" if lm > seen_ts + 1 else "ok"
+
+
+def check_enf(probe=None, targets=None, marker_path=None):
+    """HEAD every official ENF chapter. The last-seen time is the later of
+    the cached PDF's mtime (re-ingesting with --force moves it) and a date
+    recorded by --ack-guidance. When neither exists, the first run records
+    the upstream date as the baseline instead of alarming."""
+    probe = probe or _powershell_probe
+    targets = enf_targets() if targets is None else targets
+    got = probe([t["url"] for t in targets], "HEAD") if targets else {}
+    marker = _guidance_marker(marker_path)
+    seeded, rows = False, []
+    for t in targets:
+        p = got.get(t["url"]) or {"status": 0, "last_modified": "", "error": "not probed"}
+        cache = Path(t["cache"]) if t.get("cache") else None
+        seen = [x for x in ((cache.stat().st_mtime if cache and cache.exists() else None),
+                            _http_ts(marker.get(t["url"]))) if x is not None]
+        seen_ts = max(seen) if seen else None
+        status = enf_status(p, seen_ts)
+        if status == "baseline":
+            marker[t["url"]] = p["last_modified"]
+            seeded, status, seen_ts = True, "ok", _http_ts(p["last_modified"])
+        rows.append({"chapter": t["chapter"], "lang": t["lang"], "url": t["url"],
+                     "status": status, "remote": p.get("last_modified", ""),
+                     "seen": _iso_ts(seen_ts), "error": (p.get("error") or "")[:160]})
+    if seeded:
+        _write_guidance_marker(marker, marker_path)
+    return rows
+
+
+def pdi_index_links(src, html):
+    """The in-scope links a PDI category index lists (canlex.pdi's crawl rule
+    at depth 1). An empty set for a single-page set (no scope)."""
+    from . import pdi
+    if not src.get("scope"):
+        return set()
+    out = set()
+    for href in pdi._LINK.findall(pdi._main(html or "")):
+        if src["scope"] not in href or any(x in href for x in src.get("exclude", ())):
+            continue
+        out.add(href if href.startswith("http") else "https://www.canada.ca" + href)
+    return out
+
+
+def pdi_index_diff(src, live_html, seen_html="", stored_urls=(), stored_date="",
+                   today=None):
+    """One PDI category index, live against last seen. Pure. The last-seen
+    index is the one cached at ingest; without it, the stored page URLs and
+    the index chunk's date. Returns {status, new, removed, remote, seen}, where
+    status is 'changed' (pages listed or dropped), 'updated' (dcterms.modified
+    later), 'no-date' or 'ok'. 'removed' needs the cached index: stored URLs
+    include depth-2 pages that the index never lists."""
+    from . import pdi
+    live = pdi_index_links(src, live_html)
+    if seen_html:
+        base, seen = pdi_index_links(src, seen_html), pdi.page_modified(seen_html, today)
+    else:
+        base, seen = set(stored_urls), stored_date or ""
+    new = sorted(live - base)
+    removed = sorted(base - live) if seen_html else []
+    remote = pdi.page_modified(live_html or "", today)
+    if new or removed:
+        status = "changed"
+    elif not remote:
+        status = "no-date"
+    elif seen and remote > seen:
+        status = "updated"
+    else:
+        status = "ok"
+    return {"status": status, "new": new, "removed": removed, "remote": remote,
+            "seen": seen}
+
+
+def _pdi_urls():
+    from . import pdi
+    return ([s["index"] for s in pdi.SETS]
+            + [pdi._OBM + rel for _, _, rel in _PDI_KEY_PAGES])
+
+
+def check_pdi(got, today=None):
+    """Rows for each PDI category index and each key page, from probed GETs."""
+    from . import pdi
+    stored_urls, stored_dates = {}, {}
+    if pdi.OUT_FILE.exists():
+        for c in json.loads(pdi.OUT_FILE.read_text(encoding="utf-8")):
+            stored_urls.setdefault(c.get("act_code", "").lower(), set()).add(c["source_url"])
+            stored_dates.setdefault(c["source_url"], c.get("current_to", ""))
+    rows = []
+
+    def failed(name, code, url, p):
+        rows.append({"name": name, "code": code, "url": url,
+                     "status": "retired" if p.get("status") == 404 else "error",
+                     "new": [], "removed": [], "remote": "", "seen": "",
+                     "error": (p.get("error") or "no page")[:160]})
+
+    for s in pdi.SETS:
+        p = got.get(s["index"]) or {}
+        if p.get("status") != 200 or not p.get("html"):
+            failed(s["short"], s["code"], s["index"], p)
+            continue
+        cache = pdi.RAW / f"{s['code']}-index.html"
+        seen_html = cache.read_bytes().decode("utf-8", "replace") if cache.exists() else ""
+        d = pdi_index_diff(s, p["html"], seen_html, stored_urls.get(s["code"], ()),
+                           stored_dates.get(s["index"], ""), today)
+        rows.append(dict(d, name=s["short"], code=s["code"], url=s["index"], error=""))
+    for code, name, rel in _PDI_KEY_PAGES:
+        url = pdi._OBM + rel
+        p = got.get(url) or {}
+        if p.get("status") != 200 or not p.get("html"):
+            failed(name, code, url, p)
+            continue
+        stored = stored_dates.get(url, "")
+        remote = pdi.page_modified(p["html"], today)
+        status = ("no-date" if not remote else
+                  "updated" if stored and remote > stored else "ok")
+        rows.append({"name": name, "code": code, "url": url, "status": status,
+                     "new": [], "removed": [], "remote": remote, "seen": stored,
+                     "error": ""})
+    return rows
+
+
+_IRB_TITLE_ARCHIVED = re.compile(
+    r'dcterms\.title"[^>]*content="\s*ARCHIVED\b|<h1[^>]*>\s*(?:<[^>]+>\s*)*ARCHIVED\b',
+    re.I)
+_IRB_BANNER = re.compile(r'id="archived"', re.I)
+_PAGE_MODIFIED = re.compile(
+    r'dcterms\.modified"[^>]*content="(\d{4}-\d{2}-\d{2})"|'
+    r'dateModified"[^>]*>(\d{4}-\d{2}-\d{2})<')
+
+
+def irb_archive_status(html, seen):
+    """(status, modified, marker) for an IRB legal-concepts index page. Pure.
+    status: 'unarchived' (neither the 'ARCHIVED' title nor the archived-
+    content banner is left, so the chunks' 'archived' label may now be
+    wrong), 'updated' (date modified later than `seen`), 'no-date' or 'ok'."""
+    marker = {"title": bool(_IRB_TITLE_ARCHIVED.search(html or "")),
+              "banner": bool(_IRB_BANNER.search(html or ""))}
+    m = _PAGE_MODIFIED.search(html or "")
+    modified = (m.group(1) or m.group(2)) if m else ""
+    if not (marker["title"] or marker["banner"]):
+        status = "unarchived"
+    elif not modified:
+        status = "no-date"
+    elif seen and modified > seen:
+        status = "updated"
+    else:
+        status = "ok"
+    return status, modified, marker
+
+
+def _irb_urls():
+    from . import irb_papers
+    return sorted({p["index"] for p in irb_papers.PAPERS})
+
+
+def check_irb(got, marker_path=None):
+    """Rows for the IRB legal-concepts index pages, from probed GETs. Last
+    seen: a date recorded by --ack-guidance, else irb_papers.ARCHIVED (the
+    'Date modified' both pages carried when the papers were ingested)."""
+    from . import irb_papers
+    marker = _guidance_marker(marker_path)
+    rows = []
+    for url in _irb_urls():
+        papers = [p["label"] for p in irb_papers.PAPERS if p["index"] == url]
+        seen = marker.get("irb:" + url) or irb_papers.ARCHIVED
+        p = got.get(url) or {}
+        page = url.rsplit("/", 1)[-1]
+        if p.get("status") != 200 or not p.get("html"):
+            rows.append({"page": page, "url": url, "papers": papers,
+                         "status": "error", "modified": "", "seen": seen,
+                         "marker": {}, "error": (p.get("error") or "no page")[:160]})
+            continue
+        status, modified, marker_found = irb_archive_status(p["html"], seen)
+        rows.append({"page": page, "url": url, "papers": papers, "status": status,
+                     "modified": modified, "seen": seen, "marker": marker_found,
+                     "error": ""})
+    return rows
+
+
+def guidance_forms(policy_rows):
+    """The IMM 1444 and Guide 5312 rows of check_policy (their page
+    dateModified), so the guidance section names them without a second fetch."""
+    from .conduct import DOCS
+    shorts = {d["short"] for d in DOCS
+              if d["code"] in _GUIDANCE_FORMS and d.get("lang", "en") == "en"}
+    return [dict(r) for r in policy_rows or []
+            if r.get("short") in shorts and r.get("lang", "en") == "en"]
+
+
+def check_guidance(policy_rows=None, probe=None, today=None, marker_path=None,
+                   enf=None):
+    """The guidance-currency watch (roadmap T4.8): {enf, pdi, irb, forms}.
+    Two PowerShell processes run, one for the ENF HEADs and one for the PDI
+    and IRB page GETs. `enf` may pass pre-built chapter targets (tests)."""
+    probe = probe or _powershell_probe
+    today = today or datetime.date.today().isoformat()
+    enf_rows = check_enf(probe, targets=enf, marker_path=marker_path)
+    got = probe(_pdi_urls() + _irb_urls(), "GET")
+    return {"enf": enf_rows, "pdi": check_pdi(got, today),
+            "irb": check_irb(got, marker_path), "forms": guidance_forms(policy_rows)}
+
+
+def guidance_flags(guidance):
+    """[(kind, row)] of the rows that fail the check. The forms rows are not
+    counted here: check_policy's own STALE rows already count them."""
+    return [(kind, r) for kind, statuses in _GUIDANCE_FLAGS.items()
+            for r in (guidance or {}).get(kind) or [] if r["status"] in statuses]
+
+
+def ack_guidance(probe=None, marker_path=None, enf=None):
+    """Record the current upstream date of every ENF chapter flagged CHECK
+    and every IRB index page flagged UPDATED as seen. Run it after someone
+    has looked and found nothing to re-ingest. An UNARCHIVED page cannot be
+    acknowledged: the chunks' archived label has to change in
+    canlex.irb_papers."""
+    probe = probe or _powershell_probe
+    enf_rows = check_enf(probe, targets=enf, marker_path=marker_path)
+    irb_rows = check_irb(probe(_irb_urls(), "GET"), marker_path)
+    marker = _guidance_marker(marker_path)
+    acked = []
+    for r in enf_rows:
+        if r["status"] == "check" and r["remote"]:
+            marker[r["url"]] = r["remote"]
+            acked.append(f"ENF {r['chapter']} ({r['lang']}): {r['remote']}")
+    for r in irb_rows:
+        if r["status"] == "updated" and r["modified"]:
+            marker["irb:" + r["url"]] = r["modified"]
+            acked.append(f"IRB {r['page']}: {r['modified']}")
+    if acked:
+        _write_guidance_marker(marker, marker_path)
+    print("acknowledged: " + ("; ".join(acked) if acked else "nothing flagged"))
+    return 0
+
+
+_CIU_DECISIONS = "https://ciu-sdi.ca/en/important-decisions/"
+_CIU_DEC_MARKER = PROCESSED_DIR.parent / "raw" / "ciu" / "decisions-page-date.txt"
+
+
+def check_ciu_decisions():
+    """New-case-law tripwire: the CIU curates its Important Decisions page,
+    so its dateModified moving is a cheap signal that a new decision worth
+    ingesting has issued. This is the only automated freshness signal the
+    case-law corpus has -- courts publish no 'new leading case' feed -- and
+    it complements the quarterly gap-audit ritual rather than replacing it."""
+    try:
+        req = urllib.request.Request(
+            _CIU_DECISIONS, headers={"User-Agent": BROWSER_UA})
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            html = resp.read().decode("utf-8", "replace")
+        m = re.search(r'"dateModified":"(\d{4}-\d{2}-\d{2})', html)
+        remote = m.group(1) if m else ""
+    except Exception as exc:
+        return {"stored": "", "remote": "", "status": "error",
+                "error": f"{type(exc).__name__}: {exc}"}
+    if not remote:
+        return {"stored": "", "remote": "", "status": "no-date", "error": ""}
+    stored = (_CIU_DEC_MARKER.read_text(encoding="utf-8").strip()
+              if _CIU_DEC_MARKER.exists() else "")
+    if not stored:
+        # First run: record the baseline rather than alarming (atomically: a
+        # truncated marker would read as a different, older date).
+        atomic_write_text(_CIU_DEC_MARKER, remote)
+        stored = remote
+    status = "stale" if remote > stored else "ok"
+    return {"stored": stored, "remote": remote, "status": status, "error": ""}
+
+
+def check_union():
+    """Upstream drift for the CIU/PSAC documents (incl. every site VSSA).
+
+    The union reposts amended documents at the same URLs, so the drift
+    signal is each PDF's Last-Modified header against our cached copy's
+    fetch time. New/removed VSSAs surface as fetch errors or new links on
+    the next canlex.ciu run; this check covers content changes to what we
+    already hold.
+
+    Clearing a stale row needs `canlex.ciu --force`: a plain run reuses
+    the cached PDF, so it would rebuild identical chunks and leave the
+    cache mtime -- and therefore this comparison -- unchanged."""
+    import email.utils
+    from . import ciu
+    rows = []
+    targets = [(short, lang, ciu.RAW / f"{code}{'-fr' if lang == 'fr' else ''}.pdf", url)
+               for code, short, name, url, lang in ciu.DOCS]
+    try:
+        targets += [(f"VSSA {stem}", "en",
+                     ciu.RAW / f"vssa-{stem.lower()}.pdf", url)
+                    for url, stem in ciu._vssa_links()]
+    except Exception as exc:
+        rows.append({"short": "VSSA index", "lang": "en", "status": "error",
+                     "remote": "", "error": f"{type(exc).__name__}: {exc}"})
+    for short, lang, cache, url in targets:
+        try:
+            req = urllib.request.Request(url, method="HEAD",
+                                         headers={"User-Agent": BROWSER_UA})
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                lm = resp.headers.get("Last-Modified", "")
+        except Exception as exc:
+            rows.append({"short": short, "lang": lang, "status": "error",
+                         "remote": "", "error": f"{type(exc).__name__}: {exc}"})
+            continue
+        if not lm:
+            rows.append({"short": short, "lang": lang, "status": "no-date",
+                         "remote": "", "error": ""})
+            continue
+        remote_ts = email.utils.parsedate_to_datetime(lm).timestamp()
+        local_ts = cache.stat().st_mtime if cache.exists() else 0
+        rows.append({"short": short, "lang": lang,
+                     "status": "stale" if remote_ts > local_ts else "ok",
+                     "remote": lm[:25], "error": ""})
+        time.sleep(0.3)
+    return rows
+
+# Curated datasets age: state legislatures amend deferred-adjudication
+# statutes every session, so a reviewed date older than this many days earns
+# a warning even though nothing upstream can be diffed automatically.
+_CURATED_MAX_AGE_DAYS = 180
+
+
+def check_curated(today):
+    """Age-check the curated datasets and detect edit-without-regen drift.
+
+    Two signals: (1) each data/curated/*.json 'reviewed' date older than
+    _CURATED_MAX_AGE_DAYS -> aging (re-verify against the state sources);
+    (2) the newest commentary.json current_to differing from the newest
+    curated 'reviewed' -> the curated file was edited without rerunning
+    canlex.commentary + canlex.embed, so production serves stale chunks."""
+    from .commentary import CURATED, EQUIV
+    from .config import DATA_DIR
+    # Only the files canlex.commentary renders can put commentary.json out
+    # of date; a curated file the renderer never reads (transborder.json, the
+    # IRPR s. 19(1)(c)-(g) table read at run time) is age-checked only.
+    rendered = {CURATED.name, EQUIV.name}
+    rows = []
+    reviewed_by_file = {}
+    for path in sorted((DATA_DIR / "curated").glob("*.json")):
+        try:
+            reviewed = json.loads(
+                path.read_text(encoding="utf-8")).get("reviewed", "")
+        except Exception as exc:
+            rows.append({"file": path.name, "reviewed": "",
+                         "status": "error", "error": f"{type(exc).__name__}: {exc}"})
+            continue
+        if path.name in rendered:
+            reviewed_by_file[path.name] = reviewed or ""
+        status = "ok"
+        if reviewed:
+            age = (datetime.date.fromisoformat(today)
+                   - datetime.date.fromisoformat(reviewed)).days
+            if age > _CURATED_MAX_AGE_DAYS:
+                status = "aging"
+        rows.append({"file": path.name, "reviewed": reviewed,
+                     "status": status, "error": ""})
+    commentary = PROCESSED_DIR / "commentary.json"
+    newest_reviewed = max(reviewed_by_file.values(), default="")
+    if newest_reviewed and commentary.exists():
+        chunks = json.loads(commentary.read_text(encoding="utf-8"))
+        # Per rendered file, by chunk id prefix (canlex.commentary stamps
+        # every chunk with its source file's 'reviewed' as current_to):
+        # us_dispositions.json renders commentary-disp-* and commentary-state-*,
+        # us_equivalency.json commentary-equiv-*; the commentary-method-* chunks
+        # come from both and are left to the newest-vs-newest row below. Until
+        # 2026-09-21 only the two maxima were compared, so editing the older
+        # file (its reviewed still below the other's) without re-rendering
+        # printed no DRIFT row while commentary.json kept the pre-edit chunks.
+        for name, prefixes in commentary_groups().items():
+            reviewed = reviewed_by_file.get(name)
+            dates = {c.get("current_to", "") for c in chunks
+                     if str(c.get("id", "")).startswith(prefixes)}
+            if reviewed is None or not dates:
+                continue
+            newest = max(dates)
+            if newest != reviewed or len(dates) > 1:
+                rows.append({"file": "commentary.json",
+                             "reviewed": newest, "status": "drift",
+                             "error": f"{name} reviewed {reviewed} but its chunks "
+                                      f"say {', '.join(sorted(dates)) or '?'} -- rerun "
+                                      f"canlex.commentary && canlex.embed"})
+        newest_chunk = max((c.get("current_to", "") for c in chunks), default="")
+        if newest_chunk != newest_reviewed:
+            rows.append({"file": "commentary.json",
+                         "reviewed": newest_chunk, "status": "drift",
+                         "error": f"curated reviewed {newest_reviewed} but "
+                                  f"chunks say {newest_chunk or '?'} -- rerun "
+                                  f"canlex.commentary && canlex.embed"})
+    return rows
+
+
+def commentary_groups():
+    """{rendered curated file name: the id prefixes of the commentary chunks
+    it renders} -- the grouping check_curated's per-file drift row relies on
+    (tests/test_refresh.py pins it against the renderer's ids)."""
+    from .commentary import CURATED, EQUIV
+    return {CURATED.name: ("commentary-disp-", "commentary-state-"),
+            EQUIV.name: ("commentary-equiv-",)}
+
+
+def check_curated_sections(today, write_drafts=True, draft_dir=None):
+    """Every curated verify phrase against the corpus now (roadmap T4.3).
+    The rows report BROKEN (the phrase or section is gone) and AMENDED (the
+    section's last_amended is later than last_amended_seen, the date in the
+    corpus of the approved commit). A DRAFT penalty_history row goes to
+    data/curated/drafts/ when an amendment changed a quoted maximum that no
+    history row covers. The result is canlex.curated_checks.run_checks's
+    dict: {baseline, rows, drafts, draft_file, drift, error}."""
+    from . import curated_checks
+    from .config import DATA_DIR
+    return curated_checks.run_checks(curated_dir=DATA_DIR / "curated", today=today,
+                                     draft_dir=draft_dir, write=write_drafts)
+
+
+def curated_sections_flags(result):
+    """What fails the check: a broken phrase, a drafted (uncovered) history
+    row, a maximum the table does not quote (drift), or an unreadable table.
+    An amendment that left the quoted maximum alone is printed as RE-READ
+    and does not count."""
+    if result.get("error"):
+        return [{"error": result["error"]}]
+    return ([r for r in result.get("rows") or [] if r["status"] == "broken"]
+            + list(result.get("drafts") or []) + list(result.get("drift") or []))
+
+
+def _years(m):
+    return "life" if m is not None and m >= 99 else f"{m} years"
+
+
+def _print_curated_sections(res):
+    from .curated_checks import where
+    rows = res.get("rows") or []
+    base = res.get("baseline") or ""
+    print(f"\nCurated verify phrases ({len(rows)} rows, us_equivalency.json + "
+          f"transborder.json; last seen = the corpus of the approved commit "
+          f"{base[:7] or '(unreadable: amendments not checked)'}):")
+    if res.get("error"):
+        print(f"  ERROR  {res['error'][:70]}")
+        return
+    broken = [r for r in rows if r["status"] == "broken"]
+    for r in broken:
+        print(f"  BROKEN {where(r)}: {r['act']} s. {r['section']} -- {r['reason']}: "
+              f"{r['contains'][:60]!r}")
+    for pid, act, sec, new in res.get("drift") or []:
+        print(f"  NOT QUOTED {pid}: {act} s. {sec} now states "
+              f"{' and '.join(_years(m) for m in new)}, which the approved corpus did "
+              f"not state and the table does not quote")
+    for d in res.get("drafts") or []:
+        pr = d["proposed_row"]
+        print(f"  AMENDED {d['pairing']}{' limb ' + d['limb'] if d['limb'] else ''}: "
+              f"{d['act']} s. {d['section']} amended {d['amended']} "
+              f"({d['amended_by'] or '?'}; last seen {d['last_amended_seen']}): "
+              f"maximum {', '.join(_years(m) for m in d['maxima_before']) or 'none'} -> "
+              f"{', '.join(_years(m) for m in d['maxima_after']) or 'none'}; DRAFT "
+              f"history row until {pr['until']} at "
+              f"{_years(pr['max_years']) if pr['max_years'] is not None else '? (read the old version)'}")
+        for c in d.get("cautions") or []:
+            print(f"         CAUTION {c}")
+        if d.get("prose_names_date"):
+            print(f"         (the curated prose already names this amendment; only the "
+                  f"structured row is missing)")
+    covered = sorted({(r["owner"], r["act"], r["section"], ", ".join(r["covered_by"]))
+                      for r in rows if r.get("covered_by")})
+    for pid, act, sec, until in covered:
+        print(f"  ok     {pid}: {act} s. {sec} amended; the change to its maximum is "
+              f"covered by the history row until {until}")
+    answered = {(r["act"], r["section"]) for r in rows if r.get("subsections")}
+    reread = sorted({(r["act"], r["section"]) for r in rows
+                     if r.get("amended") and not r.get("subsections")} - answered)
+    if reread:
+        print(f"  RE-READ {len(reread)} section(s) amended since the approved corpus "
+              f"with no change to the maximum they are quoted for: "
+              + ", ".join(f"{a} s. {s}" for a, s in reread))
+    created = sorted({(r["act"], r["section"], l) for r in rows
+                      for l in r.get("provisions_added") or []})
+    repealed = sorted({(r["act"], r["section"], l) for r in rows
+                       for l in r.get("provisions_removed") or []})
+    if created or repealed:
+        print("  RE-READ provisions created / repealed where a verify phrase occurs: "
+              + ", ".join([f"{a} s. {s}({l}) new" for a, s, l in created]
+                          + [f"{a} s. {s}({l}) repealed" for a, s, l in repealed]))
+    if res.get("draft_file"):
+        print(f"  -> drafts for review: {res['draft_file']}")
+    if broken or res.get("drafts") or res.get("drift"):
+        print("  -> the curated JSON is HELD FOR REVIEW: put these rows to the "
+              "maintainer; do not edit data/curated/*.json from this check")
+    else:
+        print("  every phrase is in its section; no uncovered change to a quoted maximum")
+
+
+def _print_guidance(g):
+    print("\nGuidance currency (ENF chapters, PDI indexes, IRB papers, "
+          "rehabilitation forms):")
+    enf = g.get("enf") or []
+    for r in enf:
+        tag = f"ENF {r['chapter']}{' (fr)' if r['lang'] == 'fr' else ''}"
+        fr = " --fr" if r["lang"] == "fr" else ""
+        if r["status"] == "check":
+            print(f"  CHECK  {tag}: upstream Last-Modified {r['remote']} after last "
+                  f"seen {r['seen']} -- run py -m canlex.enf{fr} --force && py -m "
+                  f"canlex.embed (or, when nothing changed, py -m canlex.refresh "
+                  f"--ack-guidance)")
+        elif r["status"] == "retired":
+            print(f"  RETIRED {tag}: 404 upstream -- IRCC withdrew it; run py -m "
+                  f"canlex.enf{fr} (a 404 chapter is dropped) and report what "
+                  f"replaced it")
+        elif r["status"] in ("error", "no-date"):
+            print(f"  {'ERROR ' if r['status'] == 'error' else '?     '} {tag}: "
+                  f"{r['error'][:55] or 'no Last-Modified header'}")
+    n_ok = sum(1 for r in enf if r["status"] == "ok")
+    print(f"  ENF: {n_ok} of {len(enf)} official chapter editions unchanged since "
+          f"last seen (ENF 2 and 14 skipped: withdrawn, unofficial ATIP copies)")
+    pdi_rows = g.get("pdi") or []
+    for r in pdi_rows:
+        if r["status"] == "changed":
+            names = [u.rsplit("/", 1)[-1] for u in r["new"]][:4]
+            gone = [u.rsplit("/", 1)[-1] for u in r["removed"]][:4]
+            print(f"  CHANGED {r['name']}: index lists {len(r['new'])} new page(s)"
+                  f"{' ' + ', '.join(names) if names else ''} and dropped "
+                  f"{len(r['removed'])}{' ' + ', '.join(gone) if gone else ''} -- run "
+                  f"py -m canlex.pdi --force && py -m canlex.embed")
+        elif r["status"] == "updated":
+            print(f"  UPDATED {r['name']}: modified {r['remote']} > last seen "
+                  f"{r['seen']} -- run py -m canlex.pdi --force && py -m canlex.embed")
+        elif r["status"] == "retired":
+            print(f"  RETIRED {r['name']}: 404 upstream ({r['url']})")
+        elif r["status"] in ("error", "no-date"):
+            print(f"  {'ERROR ' if r['status'] == 'error' else '?     '} {r['name']}: "
+                  f"{r['error'][:55] or 'no dcterms.modified'}")
+    n_ok = sum(1 for r in pdi_rows if r["status"] == "ok")
+    print(f"  PDI: {n_ok} of {len(pdi_rows)} index/key pages unchanged")
+    for r in g.get("irb") or []:
+        label = f"IRB {r['page']} ({', '.join(r['papers'])})"
+        if r["status"] == "unarchived":
+            print(f"  UNARCHIVED {label}: the ARCHIVED marker is gone -- the chunks "
+                  f"still say 'archived' (canlex.irb_papers); re-read the page and "
+                  f"report it (not a refresh)")
+        elif r["status"] == "updated":
+            print(f"  UPDATED {label}: date modified {r['modified']} > last seen "
+                  f"{r['seen']} -- re-read; py -m canlex.irb_papers --force && py -m "
+                  f"canlex.embed if a paper changed, then py -m canlex.refresh "
+                  f"--ack-guidance")
+        elif r["status"] == "error":
+            print(f"  ERROR  {label}: {r['error'][:55]}")
+        else:
+            print(f"  ok     {label}: still archived (modified "
+                  f"{r['modified'] or '?'})")
+    for r in g.get("forms") or []:
+        print(f"  {'STALE ' if r['status'] == 'stale' else r['status'][:6]:6} "
+              f"{r['short']}: page dateModified {r.get('remote') or r.get('stored') or '?'}"
+              f" (read by the employer-policy check above)")
+
+
+def check_tariff_edition(today):
+    """Probe whether CBSA has published a Customs Tariff edition newer than
+    the ingested one. The tariff pages live under a year-stamped path
+    (.../tariff-tarif/<year>/html/...), so probing the ingested-edition-plus-
+    one URL (and the current calendar year's) is a reliable new-edition
+    signal without parsing anything."""
+    from . import tariff_schedule
+    src = next(iter(tariff_schedule.SOURCES.values()))
+    have = int(src["edition"])
+    candidates = sorted({have + 1, int(today[:4])} - {have})
+    for year in candidates:
+        if year <= have:
+            continue
+        probe = src["url"].replace(f"/{src['edition']}/", f"/{year}/")
+        try:
+            req = urllib.request.Request(
+                probe, method="HEAD", headers={"User-Agent": BROWSER_UA})
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                if resp.status == 200:
+                    return {"stored": str(have), "remote": str(year),
+                            "status": "stale", "error": ""}
+        except urllib.error.HTTPError as exc:
+            if exc.code == 404:
+                continue                   # 404 = that edition doesn't exist yet
+            return {"stored": str(have), "remote": "", "status": "error",
+                    "error": f"HTTP {exc.code} probing {probe}"}
+        except Exception as exc:
+            # DNS down, a timeout, a TLS refusal: not 'ok' -- until 2026-09-21
+            # every exception read as 'no newer edition'.
+            return {"stored": str(have), "remote": "", "status": "error",
+                    "error": f"{type(exc).__name__}: {exc}"}
+    return {"stored": str(have), "remote": "", "status": "ok", "error": ""}
+
+
+def dmemo_drift(raw, stored):
+    """(newest active LAST_UPDATED, memos newer than `stored`) from the CBSA
+    D-memo datasource JSON. Pure, for testability."""
+    rows = [r for r in json.loads(raw).get("data", [])
+            if str(r.get("STATUS", "")).strip().lower() != "cancelled"]
+    remote = max((r.get("LAST_UPDATED", "") for r in rows), default="")
+    drifted = sorted(_dmemo_number(r) for r in rows
+                     if stored and r.get("LAST_UPDATED", "") > stored)
+    return remote, drifted
+
+
+def _dmemo_number(row):
+    """'D1-2-1' out of a datasource NUMBER cell (an anchor, or bare text)."""
+    cell = str(row.get("NUMBER", ""))
+    inner = re.search(r">([^<]+)<", cell)
+    return (inner.group(1) if inner else re.sub(r"<[^>]+>", "", cell)).strip()
+
+
+def check_dmemo_index():
+    """Newest per-memo 'Last updated' upstream vs stored currency.
+
+    This reads the JSON datasource behind the index table, NOT dates in the
+    index HTML: since CBSA rebuilt the page (2026-07-10) the newest date in the
+    markup is the page's own 'Date modified', which moves when the page is
+    rebuilt and no memo has changed -- that would report stale forever. The
+    datasource carries a LAST_UPDATED per memo, so drift is both true and named.
+    """
+    stored = ""
+    path = PROCESSED_DIR / "dmemos.json"
+    if path.exists():
+        chunks = json.loads(path.read_text(encoding="utf-8"))
+        stored = max((c.get("current_to", "") for c in chunks), default="")
+    try:
+        req = urllib.request.Request(
+            dmemo.DATASOURCE_URL, headers={"User-Agent": BROWSER_UA})
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            remote, drifted = dmemo_drift(resp.read(), stored)
+    except Exception as exc:
+        return {"stored": stored, "remote": "", "drifted": [], "status": "error",
+                "error": f"{type(exc).__name__}: {exc}"}
+    status = "stale" if (remote and stored and remote > stored) else \
+        ("ok" if remote else "no-date")
+    return {"stored": stored, "remote": remote, "drifted": drifted,
+            "status": status, "error": ""}
+
+# NJC directive dates as published: the index prints prose ('June  1, 1993' --
+# double spaces occur in the raw markup) while chunks may store either that
+# prose or the ISO fallback directive.py derives from page text, so both sides
+# are normalized to ISO before comparing.
+_NJC_PROSE_DATE = re.compile(r"([A-Za-z]+)\s+(\d{1,2}),?\s+(\d{4})")
+_ISO_DATE = re.compile(r"\d{4}-\d{2}-\d{2}")
+
+# CBSA publishes its IRPA delegation as a base instrument plus incremental
+# amendments (no consolidation), each a dated 'irpa-lipr-YYYY-MM-DD-eng.html'
+# linked from the CBSA legislation index. A date on the index that we have not
+# ingested means a new amendment to add to delegation.SOURCES.
+_CBSA_INDEX_URL = ("https://www.cbsa-asfc.gc.ca/agency-agence/actreg-loireg/"
+                   "legislation-eng.html")
+_CBSA_INSTRUMENT_RE = re.compile(r"irpa-lipr-(\d{4}-\d{2}-\d{2})-eng\.html")
+# The IRCC IL3 instrument is reissued with a seasonal version stamp on page 1.
+_IL3_VERSION_RE = re.compile(r"(?:Spring|Summer|Fall|Winter)\s+\d{4}")
+
+# Bills known to be heading into force that affect the corpus. Surfaced once
+# their in-force date has passed, with the codes to re-ingest. (The legislation
+# drift check above will independently flag the affected Act once Justice Laws
+# consolidates the amendment; this list just names the cause.)
+_PENDING_BILLS = [
+    {"bill": "C-16", "name": "Protecting Victims Act",
+     "in_force": "2026-07-18", "affects": ["C-46"],
+     "note": "Criminal Code: coercive control, child-protection reporting "
+             "(coercive-control provisions come into force later)."},
+]
+
+
+def _extract_current_date(raw):
+    """Pull the lims:current-date (YYYY-MM-DD) from the head of a Justice Laws
+    XML document. Pure and offline, for testability. Returns '' if absent."""
+    if raw[:3] == b"\xef\xbb\xbf":
+        raw = raw[3:]
+    m = _CURRENT_DATE.search(raw)
+    return m.group(1).decode("ascii") if m else ""
+
+
+def _stored_current_to(fname):
+    """The current_to stored in data/processed/<fname> (first chunk), '' when
+    the file is empty, None when it does not exist."""
+    path = PROCESSED_DIR / fname
+    if not path.exists():
+        return None
+    chunks = json.loads(path.read_text(encoding="utf-8"))
+    return chunks[0].get("current_to", "") if chunks else ""
+
+
+def _local_current_to(code):
+    """The current_to stored in data/processed/<code>.json (first chunk), or ''."""
+    return _stored_current_to(f"{code}.json")
+
+
+def check_bilingual(codes=None, reader=_stored_current_to):
+    """Every Justice Laws instrument must carry its French twin at the SAME
+    consolidation date. The legislation check only reads the English file,
+    so an instrument ingested in one language -- or refreshed in one
+    language only -- would pass it silently. Pure given `reader`, for tests."""
+    rows = []
+    for code in (codes or list(SOURCES)):
+        en = reader(f"{code}.json")
+        fr = reader(f"{code}-fr.json")
+        if en is None:
+            status = "missing-en"          # check_legislation reports this too
+        elif fr is None:
+            status = "missing-fr"
+        elif (fr or "") != (en or ""):
+            status = "fr-drift"
+        else:
+            status = "ok"
+        rows.append({"code": code, "en": en or "", "fr": fr or "",
+                     "status": status})
+    return rows
+
+
+# Processed files no watch above is responsible for. Every corpus family must
+# be claimed by SOURCES (legislation, EN + FR), _NON_XML, or a dedicated
+# upstream check -- otherwise a new family can be added to the corpus and
+# never be looked at again. Pure, for tests.
+_DEDICATED = {"directives.json": "check_directives",
+              "delegation.json": "check_delegation"}
+
+
+def unwatched_files(files, sources=None, non_xml=None, dedicated=None):
+    sources = list(SOURCES) if sources is None else sources
+    non_xml = _NON_XML if non_xml is None else non_xml
+    dedicated = _DEDICATED if dedicated is None else dedicated
+    claimed = set(dedicated)
+    for code in sources:
+        claimed.add(f"{code}.json")
+        claimed.add(f"{code}-fr.json")
+    claimed.update(fname for _, fname, _, _ in non_xml)
+    return sorted(f for f in files if f.endswith(".json") and f not in claimed)
+
+
+def _remote_current_date(url):
+    """Fetch only the head of the XML and extract its consolidation date.
+    Returns (date, error): date is '' on a clean fetch that lacked the
+    attribute, error is a short string when the fetch itself failed."""
+    req = urllib.request.Request(url, headers={"User-Agent": _UA})
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            head = resp.read(_HEAD_BYTES)
+        return _extract_current_date(head), ""
+    except Exception as exc:                         # network / HTTP / timeout
+        return "", f"{type(exc).__name__}: {exc}"
+
+
+def check_legislation():
+    """Compare each Justice Laws source's upstream date to the stored corpus.
+    Returns a list of row dicts."""
+    rows = []
+    for code, src in SOURCES.items():
+        local = _local_current_to(code)
+        remote, error = _remote_current_date(src["xml_url"])
+        if error:
+            status = "error"
+        elif local is None:
+            status = "missing-local"      # configured but never ingested
+        elif remote and remote > local:
+            status = "stale"              # upstream newer than our corpus
+        elif not remote:
+            status = "no-date"            # fetched, but no date found (unusual)
+        else:
+            status = "ok"
+        rows.append({"code": code, "short": src["short"], "local": local or "",
+                     "remote": remote, "status": status, "error": error})
+        time.sleep(0.3)                   # be polite to Justice Laws
+    return rows
+
+
+def non_xml_currency():
+    """Stored currency (newest item) of the aggregate sources."""
+    rows = []
+    for label, fname, cmd, desc in _NON_XML:
+        path = PROCESSED_DIR / fname
+        if not path.exists():
+            rows.append({"label": label, "desc": desc, "cmd": cmd,
+                         "newest": "", "chunks": 0, "present": False})
+            continue
+        chunks = json.loads(path.read_text(encoding="utf-8"))
+        newest = max((c.get("current_to", "") for c in chunks), default="")
+        rows.append({"label": label, "desc": desc, "cmd": cmd,
+                     "newest": newest, "chunks": len(chunks), "present": True})
+    return rows
+
+
+def _cbsa_ingested_dates():
+    """The effective dates of the CBSA dated delegation instruments we ingest."""
+    return sorted({s["effective"] for s in delegation.SOURCES.values()
+                   if s.get("kind") == "html-cbsa" and s.get("effective")})
+
+
+def _ircc_stored_version():
+    """The IL3 version string recorded in the ingested corpus, or ''."""
+    path = PROCESSED_DIR / "delegation.json"
+    if not path.exists():
+        return ""
+    chunks = json.loads(path.read_text(encoding="utf-8"))
+    versions = [c.get("current_to", "") for c in chunks
+                if c.get("act_short") == "IRCC IL3" and c.get("current_to")]
+    return max(versions) if versions else ""
+
+
+def check_delegation():
+    """Compare the delegation instruments against upstream. Returns row dicts.
+
+    CBSA: diff the dated instruments published on the legislation index against
+    those we ingest -- a new date means an un-ingested amendment. IRCC: compare
+    the IL3 version stamp published in the live PDF against the stored one."""
+    rows = []
+
+    # --- CBSA: index diff (cheap; one HTML fetch) ---
+    have = _cbsa_ingested_dates()
+    try:
+        req = urllib.request.Request(_CBSA_INDEX_URL,
+                                     headers={"User-Agent": BROWSER_UA})
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            html = resp.read().decode("utf-8", "replace")
+        published = sorted(set(_CBSA_INSTRUMENT_RE.findall(html)))
+        new = sorted(set(published) - set(have))
+        rows.append({"name": "CBSA IRPA delegation (+ amendments)",
+                     "status": "stale" if new else "ok",
+                     "have": have, "published": published, "new": new,
+                     "error": ""})
+    except Exception as exc:
+        rows.append({"name": "CBSA IRPA delegation (+ amendments)",
+                     "status": "error", "have": have, "published": [],
+                     "new": [], "error": f"{type(exc).__name__}: {exc}"})
+
+    # --- IRCC IL3: version-stamp compare (re-fetches the PDF via PowerShell) ---
+    stored = _ircc_stored_version()
+    try:
+        with tempfile.TemporaryDirectory() as td:
+            data = delegation._fetch(delegation.SOURCES["ircc"]["url"],
+                                     Path(td) / "il3.pdf", powershell=True)
+        pages = delegation._pdf_pages(data)
+        m = _IL3_VERSION_RE.search(pages[0]) if pages else None
+        published = m.group(0) if m else ""
+        if not published:
+            status = "no-version"
+        elif published != stored:
+            status = "stale"
+        else:
+            status = "ok"
+        rows.append({"name": "IRCC IL3 instrument", "status": status,
+                     "have": [stored], "published": [published], "new": [],
+                     "error": ""})
+    except Exception as exc:
+        rows.append({"name": "IRCC IL3 instrument", "status": "error",
+                     "have": [stored], "published": [], "new": [],
+                     "error": f"{type(exc).__name__}: {exc}"})
+    return rows
+
+
+def _stored_directives():
+    """Per-directive stored currency from data/processed/directives.json:
+    {code: (title, current_to, source_url)}, one entry per act_code."""
+    path = PROCESSED_DIR / "directives.json"
+    if not path.exists():
+        return {}
+    chunks = json.loads(path.read_text(encoding="utf-8"))
+    out = {}
+    for c in chunks:
+        code = c.get("act_code")
+        if code and code not in out:
+            out[code] = (c.get("act_short", ""), c.get("current_to", ""),
+                         c.get("source_url", ""))
+    return out
+
+
+def _directive_iso(text):
+    """Normalize an NJC directive date ('June  1, 1993' or already ISO) to
+    YYYY-MM-DD. Pure, for testability. Returns '' when not a recognizable date."""
+    text = (text or "").strip()
+    if _ISO_DATE.fullmatch(text):
+        return text
+    m = _NJC_PROSE_DATE.fullmatch(text)
+    if m and m.group(1).lower() in MONTHS:
+        return (f"{int(m.group(3)):04d}-{MONTHS[m.group(1).lower()]:02d}-"
+                f"{int(m.group(2)):02d}")
+    return ""
+
+
+def _njc_index_dates(html):
+    """{code: date text} parsed from the NJC directive index, mirroring
+    directive.directive_links: the first <a> in each top-level li of
+    ul.directive-list is the current directive (nested lis are archived
+    versions), and the sibling span.date its effective date ('' if absent)."""
+    soup = BeautifulSoup(html, "html.parser")
+    ul = soup.find("ul", class_="directive-list")
+    out = {}
+    if ul is None:
+        return out
+    for li in ul.find_all("li", recursive=False):
+        a = li.find("a", href=True)
+        if not a:
+            continue
+        m = directive._CODE.search(a["href"])
+        if not m:
+            continue
+        span = li.find("span", class_="date")
+        out[m.group(1)] = span.get_text(strip=True) if span else ""
+    return out
+
+
+def _njc_page_effective(url, td):
+    """Fresh-fetch a directive's own page and pull its 'effective <Month D,
+    YYYY>' prose (directive._page_effective_date), following the 'Print Full
+    Directive' link when the landing page is only a table of contents (the
+    FSDs). Returns (iso_date, error)."""
+    try:
+        html = directive._fetch(url, td / "probe.html", force=True)
+        soup = BeautifulSoup(html, "html.parser")
+        main = soup.find("main")
+        date = directive._page_effective_date(main) if main else ""
+        if not date:
+            print_url = directive._print_link(html)
+            if print_url:
+                full = directive._fetch(print_url, td / "probe-full.html",
+                                        force=True)
+                fmain = BeautifulSoup(full, "html.parser").find("main")
+                date = directive._page_effective_date(fmain) if fmain else ""
+        return date, ""
+    except Exception as exc:
+        return "", f"{type(exc).__name__}: {exc}"
+
+
+def check_directives():
+    """Compare each ingested NJC directive's effective date against upstream.
+    Returns row dicts.
+
+    The date directive.py stores in current_to comes from the NJC directive
+    index (the span.date beside each entry), so one fresh index fetch covers
+    nearly every directive; entries the index leaves undated (the Foreign
+    Service Directives) fall back to the 'effective <Month D, YYYY>' prose on
+    the directive's own page -- the same two signals ingestion uses. Probe
+    fetches go to a temp dir, never data/raw, so a stale probe can't mask a
+    later real refresh; directive._fetch already sleeps 0.5s after each
+    download, so the probes stay polite without extra pauses here."""
+    stored = _stored_directives()
+    rows = []
+    if not stored:
+        return rows
+    with tempfile.TemporaryDirectory() as td:
+        td = Path(td)
+        try:
+            html = directive._fetch(directive.INDEX_URL, td / "_index.html",
+                                    force=True)
+            index_dates, index_error = _njc_index_dates(html), ""
+        except Exception as exc:
+            index_dates, index_error = {}, f"{type(exc).__name__}: {exc}"
+        for code, (title, stored_date, url) in sorted(stored.items()):
+            upstream, error = index_dates.get(code, ""), index_error
+            if not upstream and not error:
+                # Undated on the index: read the date off the page itself.
+                upstream, error = _njc_page_effective(url, td)
+            local, remote = _directive_iso(stored_date), _directive_iso(upstream)
+            if error:
+                status = "error"
+            elif not remote:
+                status = "no-date"
+            elif remote != local:
+                status = "stale"      # NJC reissues in place: any drift counts
+            else:
+                status = "ok"
+            rows.append({"code": code, "title": title, "stored": stored_date,
+                         "local": local, "remote": remote, "status": status,
+                         "error": error})
+    return rows
+
+
+def pending_in_force(today):
+    """Pending bills whose in-force date has passed as of `today` (ISO str)."""
+    return [b for b in _PENDING_BILLS if b["in_force"] <= today]
+
+
+def pending_consolidation(bill, reader=None):
+    """[{code, stored, status}] for each Act `bill` affects: the consolidation
+    date stored for it and whether that already covers the bill's in-force
+    date -- 'consolidated' (stored current_to on or after it: a Justice Laws
+    consolidation current to a date includes what came into force by then),
+    'predates' (the re-ingest is still needed, once Justice Laws has
+    consolidated it) or 'absent' (not ingested). Until 2026-09-22 the notice
+    printed 'run: py -m canlex.ingest --force C-46' either way, for a Criminal
+    Code already stored current to 2026-07-21 against C-16's 2026-07-18.
+    `reader` maps a processed filename to its stored current_to
+    (_stored_current_to); pure given it, for tests."""
+    reader = reader or _stored_current_to
+    rows = []
+    for code in bill["affects"]:
+        stored = reader(f"{code}.json") or ""
+        status = ("absent" if not stored
+                  else "consolidated" if stored >= bill["in_force"]
+                  else "predates")
+        rows.append({"code": code, "stored": stored, "status": status})
+    return rows
+
+
+def pending_lines(bill, rows):
+    """The staleness report's lines for one pending bill now in force."""
+    lines = [f"  Bill {bill['bill']} ({bill['name']}) in force {bill['in_force']} "
+             f"-> {', '.join(bill['affects'])}",
+             f"     {bill['note']}"]
+    for r in rows:
+        if r["status"] == "consolidated":
+            lines.append(f"     {r['code']}: stored consolidation current to {r['stored']}, "
+                         f"on or after {bill['in_force']} -- what came into force that "
+                         f"day is in the corpus; no re-ingest needed for it")
+        elif r["status"] == "predates":
+            lines.append(f"     {r['code']}: stored consolidation current to {r['stored']} "
+                         f"predates {bill['in_force']} -- re-ingest once Justice Laws "
+                         f"has consolidated it (the legislation rows above say STALE)")
+            lines.append(f"     run: py -m canlex.ingest --force {r['code']} && "
+                         f"py -m canlex.ingest --fr --force {r['code']}")
+        else:
+            lines.append(f"     {r['code']}: not ingested -- run: py -m canlex.ingest "
+                         f"{r['code']}")
+    return lines
+
+
+def run(as_json=False):
+    today = datetime.date.today().isoformat()
+    leg = check_legislation()
+    deleg = check_delegation()
+    dirs = check_directives()
+    nonxml = non_xml_currency()
+    curated = check_curated(today)
+    dmemo_idx = check_dmemo_index()
+    tariff = check_tariff_edition(today)
+    union = check_union()
+    ciu_dec = check_ciu_decisions()
+    policy = check_policy()
+    policy_stale = [r for r in policy if r["status"] in ("stale", "missing-local")]
+    guidance = check_guidance(policy, today=today)
+    guidance_flag = guidance_flags(guidance)
+    sections = check_curated_sections(today)
+    sections_flag = curated_sections_flags(sections)
+    pending = [dict(b, consolidation=pending_consolidation(b))
+               for b in pending_in_force(today)]
+    twins = check_bilingual()
+    twins_bad = [r for r in twins if r["status"] in ("missing-fr", "fr-drift")]
+    unwatched = unwatched_files(p.name for p in PROCESSED_DIR.glob("*.json"))
+    stale = [r for r in leg if r["status"] in ("stale", "missing-local")]
+    errors = [r for r in leg if r["status"] == "error"]
+    deleg_stale = [r for r in deleg if r["status"] == "stale"]
+    dirs_stale = [r for r in dirs if r["status"] == "stale"]
+    curated_flag = [r for r in curated if r["status"] in ("aging", "drift")]
+    dmemo_stale = dmemo_idx["status"] == "stale"
+    tariff_stale = tariff["status"] == "stale"
+    union_stale = [r for r in union if r["status"] == "stale"]
+    ciu_dec_stale = ciu_dec["status"] == "stale"
+    errors_other = probe_errors(deleg, dirs, curated, union, policy, guidance,
+                                dmemo_idx, tariff, ciu_dec)
+
+    if as_json:
+        print(json.dumps({"checked": today, "legislation": leg,
+                          "delegation": deleg, "directives": dirs,
+                          "non_xml": nonxml, "curated": curated,
+                          "dmemo_index": dmemo_idx, "tariff": tariff,
+                          "union": union, "ciu_decisions": ciu_dec,
+                          "pending_in_force": pending,
+                          "bilingual": twins, "unwatched": unwatched,
+                          "policy": policy,
+                          "guidance": guidance,
+                          "curated_sections": sections,
+                          "stale_count": (len(stale) + len(deleg_stale)
+                                          + len(dirs_stale)
+                                          + len(curated_flag)
+                                          + len(union_stale)
+                                          + len(twins_bad) + len(unwatched)
+                                          + len(policy_stale)
+                                          + len(guidance_flag)
+                                          + len(sections_flag)
+                                          + (1 if dmemo_stale else 0)
+                                          + (1 if tariff_stale else 0)),
+                          "error_count": len(errors) + len(errors_other)}, indent=2))
+    else:
+        print(f"CanLex corpus staleness check — {today}\n")
+        print(f"Justice Laws legislation ({len(leg)} instruments):")
+        for r in sorted(leg, key=lambda x: (x["status"] != "stale", x["short"])):
+            if r["status"] == "stale":
+                print(f"  STALE  {r['short']:34} local {r['local']}  "
+                      f"upstream {r['remote']}")
+            elif r["status"] == "missing-local":
+                print(f"  ABSENT {r['short']:34} configured but not ingested "
+                      f"(upstream {r['remote']})")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['short']:34} {r['error'][:50]}")
+            elif r["status"] == "no-date":
+                print(f"  ?      {r['short']:34} no current-date found upstream")
+        if not stale and not errors:
+            print(f"  all {len(leg)} current (local matches or exceeds upstream)")
+        if stale:
+            codes = " ".join(r["code"] for r in stale)
+            print(f"\n  -> {len(stale)} drifted. Refresh: "
+                  f"py -m canlex.ingest --force {codes}\n"
+                  f"     and the French twins: py -m canlex.ingest --fr "
+                  f"--force {codes}\n"
+                  f"     then: py -m canlex.embed && py -m canlex.eval")
+
+        print(f"\nFrench twins ({len(twins)} instruments):")
+        for r in twins_bad:
+            if r["status"] == "missing-fr":
+                print(f"  ABSENT {r['code']:34} no French edition ingested")
+            else:
+                print(f"  DRIFT  {r['code']:34} fr {r['fr'] or '?'} != en "
+                      f"{r['en'] or '?'}")
+        if twins_bad:
+            codes = " ".join(r["code"] for r in twins_bad)
+            print(f"  -> py -m canlex.ingest --fr --force {codes} && "
+                  f"py -m canlex.embed")
+        else:
+            print(f"  all {len(twins)} present at the English consolidation "
+                  f"date")
+
+        if unwatched:
+            print("\nUNWATCHED corpus files (no staleness watch claims them --"
+                  " add the family to refresh._NON_XML, config.SOURCES or a"
+                  " dedicated check before the next refresh):")
+            for f in unwatched:
+                print(f"  !! {f}")
+
+        print("\nDelegation instruments (upstream check):")
+        for r in deleg:
+            if r["status"] == "stale" and r["new"]:
+                print(f"  STALE  {r['name']}: new upstream {', '.join(r['new'])} "
+                      f"not ingested")
+            elif r["status"] == "stale":
+                print(f"  STALE  {r['name']}: upstream {r['published'][0] or '?'} "
+                      f"!= stored {r['have'][0] or '?'}")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['name']}: {r['error'][:50]}")
+            elif r["status"] == "no-version":
+                print(f"  ?      {r['name']}: no version stamp found upstream")
+            elif len(r["published"]) > 1:        # CBSA: a set of dated instruments
+                print(f"  ok     {r['name']}: current "
+                      f"({len(r['published'])} instruments, latest "
+                      f"{max(r['published'])})")
+            else:                                # IRCC: a single version stamp
+                shown = r["published"][0] if r["published"] else "current"
+                print(f"  ok     {r['name']}: current ({shown})")
+        if deleg_stale:
+            print("     -> re-ingest: add any new CBSA dates to delegation.SOURCES, "
+                  "then py -m canlex.delegation && py -m canlex.embed")
+
+        print(f"\nNJC directives (upstream check, {len(dirs)} ingested):")
+        for r in dirs:
+            if r["status"] == "stale":
+                print(f"  STALE  {r['title']}: upstream {r['remote']} "
+                      f"!= stored {r['local'] or r['stored'] or '?'}")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['title']}: {r['error'][:50]}")
+            elif r["status"] == "no-date":
+                print(f"  ?      {r['title']}: no effective date found upstream")
+            else:
+                print(f"  ok     {r['title']}: current ({r['remote']})")
+        if dirs_stale:
+            print("     -> re-ingest: py -m canlex.directive --force && "
+                  "py -m canlex.embed")
+
+        print("\nOther sources (stored currency; re-run the ingester to refresh):")
+        for r in nonxml:
+            mark = " " if r["present"] else "!"
+            print(f" {mark}{r['label']:14} {str(r['chunks']):>5} chunks  "
+                  f"newest {r['newest'] or 'n/a':12}  {r['cmd']}")
+
+        print("\nCurated datasets (age + regen-drift check):")
+        for r in curated:
+            if r["status"] == "aging":
+                print(f"  AGING  {r['file']}: reviewed {r['reviewed']} — older "
+                      f"than {_CURATED_MAX_AGE_DAYS} days; re-verify the state "
+                      f"rows against their statutes and bump 'reviewed'")
+            elif r["status"] == "drift":
+                print(f"  DRIFT  {r['file']}: {r['error']}")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['file']}: {r['error'][:60]}")
+            else:
+                print(f"  ok     {r['file']}: reviewed {r['reviewed'] or '?'}")
+
+        _print_curated_sections(sections)
+
+        if dmemo_idx["status"] == "stale":
+            drifted = dmemo_idx.get("drifted") or []
+            named = (f" ({', '.join(drifted[:6])}"
+                     f"{', …' if len(drifted) > 6 else ''})" if drifted else "")
+            print(f"\nD-memo index: STALE — newest upstream 'Last updated' "
+                  f"{dmemo_idx['remote']} > stored {dmemo_idx['stored']}"
+                  f"{named}; "
+                  f"run py -m canlex.dmemo --force && py -m canlex.embed")
+        elif dmemo_idx["status"] == "error":
+            print(f"\nD-memo index: ERROR {dmemo_idx['error'][:60]}")
+        else:
+            print(f"\nD-memo index: ok (upstream {dmemo_idx['remote'] or '?'} "
+                  f"<= stored {dmemo_idx['stored'] or '?'})")
+
+        if tariff["status"] == "stale":
+            print(f"\nCustoms Tariff edition: STALE — CBSA has published the "
+                  f"{tariff['remote']} edition (ingested: {tariff['stored']}); "
+                  f"bump the year in tariff_schedule.SOURCES and re-ingest")
+        else:
+            print(f"\nCustoms Tariff edition: ok ({tariff['stored']} is "
+                  f"current)")
+
+        n_union_ok = sum(1 for r in union if r["status"] == "ok")
+        print(f"\nCIU/PSAC union documents (upstream Last-Modified check, "
+              f"{len(union)} documents):")
+        for r in union:
+            if r["status"] == "stale":
+                print(f"  STALE  {r['short']} ({r['lang']}): upstream "
+                      f"modified {r['remote']} after our fetch — run "
+                      f"py -m canlex.ciu --force && py -m canlex.embed")
+            elif r["status"] == "error":
+                print(f"  ERROR  {r['short']}: {r['error'][:55]}")
+        if not union_stale and not any(r["status"] == "error" for r in union):
+            print(f"  all {n_union_ok} current (upstream unchanged since fetch)")
+
+        print(f"\nEmployer policy pages (upstream date-modified check):")
+        for r in policy:
+            tag = r["short"] + (" (fr)" if r.get("lang", "en") == "fr" else "")
+            if r["status"] == "stale":
+                print(f"  STALE  {tag}: upstream modified {r['remote']} "
+                      f"> stored {r['stored']} -- run py -m canlex.conduct "
+                      f"--force && py -m canlex.embed")
+            elif r["status"] == "missing-local":
+                print(f"  ABSENT {tag}: configured but not ingested")
+            elif r["status"] == "error":
+                print(f"  ERROR  {tag}: {r['error'][:55]}")
+            else:
+                print(f"  ok     {tag}: current "
+                      f"({r['remote'] or r['stored'] or '?'})")
+
+        _print_guidance(guidance)
+
+        if ciu_dec["status"] == "stale":
+            print(f"\nCIU Important-Decisions page: UPDATED "
+                  f"({ciu_dec['remote']} > {ciu_dec['stored']}) — the union "
+                  f"may have flagged a new decision; review "
+                  f"{_CIU_DECISIONS} and, after ingesting anything new, "
+                  f"update {_CIU_DEC_MARKER.name} to {ciu_dec['remote']}")
+        elif ciu_dec["status"] == "error":
+            print(f"\nCIU Important-Decisions page: ERROR "
+                  f"{ciu_dec['error'][:55]}")
+        else:
+            print(f"\nCIU Important-Decisions page: ok (last modified "
+                  f"{ciu_dec['remote'] or '?'})")
+
+        if pending:
+            print("\nPending bills now in force (is the stored consolidation "
+                  "on or after the in-force date?):")
+            for b in pending:
+                print("\n".join(pending_lines(b, b["consolidation"])))
+
+    if errors_other and not as_json:
+        print(f"\n{len(errors_other)} probe error(s) in the other watches "
+              f"(exit 1: a watch that cannot reach its source has not checked it):")
+        for kind, what, err in errors_other[:12]:
+            print(f"  ERROR  {kind}: {what}: {err[:60]}")
+        if len(errors_other) > 12:
+            print(f"  ... and {len(errors_other) - 12} more")
+
+    return 1 if (stale or errors or errors_other or deleg_stale or dirs_stale
+                 or curated_flag or dmemo_stale or tariff_stale
+                 or union_stale or ciu_dec_stale or twins_bad
+                 or unwatched or policy_stale or guidance_flag
+                 or sections_flag) else 0
+
+
+def probe_errors(deleg, dirs, curated, union, policy, guidance,
+                 dmemo_idx, tariff, ciu_dec):
+    """[(watch, item, error)] for every row of the non-legislation watches
+    whose status is 'error'. Until 2026-09-21 only the legislation rows'
+    errors reached the exit status: a broken PowerShell probe printed 52
+    ERROR rows in the guidance watch and the check passed, and
+    check_tariff_edition read a network failure as 'ok'."""
+    out = []
+    for kind, rows, label in (("delegation", deleg, "name"), ("directives", dirs, "title"),
+                              ("curated", curated, "file"), ("union", union, "short"),
+                              ("policy", policy, "short")):
+        out += [(kind, str(r.get(label, "?")), r.get("error", "")) for r in rows or []
+                if r.get("status") == "error"]
+    for kind in ("enf", "pdi", "irb"):
+        out += [(f"guidance/{kind}", str(r.get("url") or r.get("chapter") or r.get("page") or "?"),
+                 r.get("error", "")) for r in (guidance or {}).get(kind) or []
+                if r.get("status") == "error"]
+    for kind, row in (("dmemo-index", dmemo_idx), ("tariff-edition", tariff),
+                      ("ciu-decisions", ciu_dec)):
+        if (row or {}).get("status") == "error":
+            out.append((kind, "probe", row.get("error", "")))
+    return out
+
+
+def main():
+    args = sys.argv[1:]
+    as_json = "--json" in args
+    if "--ack-guidance" in args:
+        # After someone has looked at a CHECK / UPDATED guidance row and found
+        # nothing to re-ingest: record the upstream date as seen.
+        sys.exit(ack_guidance())
+    # --check is the default mode; accepted for forward-compatibility.
+    sys.exit(run(as_json=as_json))
+
+
+if __name__ == "__main__":
+    main()

canlex/rerank.py

@@ -0,0 +1,183 @@
+"""Cross-encoder reranker over ONNX Runtime (local, key-free)."""
+import os
+
+import numpy as np
+import onnxruntime as ort
+from huggingface_hub import hf_hub_download
+from huggingface_hub.errors import LocalEntryNotFoundError
+from tokenizers import Tokenizer
+
+RERANK_REPO = "Xenova/bge-reranker-base"
+RERANK_ONNX = "onnx/model_quantized.onnx"  # int8: ~3x faster on CPU, negligible quality loss
+MAX_TOKENS = 512
+_MAX_DOC_CHARS = int(os.environ.get("CANLEX_RERANK_DOC_CHARS", "2000"))
+                    # 1000 -> 2000 adopted 2026-07-23: the doubled reading
+                    # window held every 203-Q eval metric and lifted Hit@5
+                    # +0.01; the cross-encoder now judges a full-size piece
+                    # rather than 60% of one.
+                       # cap doc text before tokenizing. Cross-encoder cost is
+                       # ~linear in tokens per doc, so halving this ~halves the
+                       # dominant query cost; env-sweepable, eval-gated (a
+                       # tighter cap scores long sections on less of their text
+                       # -- fine in practice: the median section is ~520 chars,
+                       # and sub-chunked pieces cap at 1800). Swept 2026-06:
+                       # 3000 -> 1000 (with pool 50 -> 16) held the eval exactly.
+_MAX_QUERY_TOKENS = 256
+                    # The query's token budget in a (query, document) pair.
+                    # only_second truncation keeps the query whole and RAISES
+                    # when the query alone overflows MAX_TOKENS (about 509
+                    # tokens, 400 words of English), which took the whole
+                    # search() down (code audit 2026-09-21). A query past the
+                    # budget is cut to its first _MAX_QUERY_TOKENS tokens, on
+                    # a token boundary, before pairing, so the document keeps
+                    # at least MAX_TOKENS - _MAX_QUERY_TOKENS - 4 tokens. A
+                    # query within the budget (the MCP tool's 500-character
+                    # cap is about 130 tokens of English) is scored exactly
+                    # as before.
+
+
+def hub_file(repo, filename):
+    """The local path of one model file: the Hugging Face cache first, the
+    network only when the file is not cached (LocalEntryNotFoundError).
+
+    A plain hf_hub_download revalidates every file against huggingface.co
+    even when it is cached: 6-7 HEAD requests per cold start, the first tool
+    call 12.5-13.3 s online, 24-48 s with the network refused and 106 s
+    black-holed (code audit 2026-09-21, perf-07). A cached file is now read
+    with no request at all; a fresh clone still downloads once. Under
+    HF_HUB_OFFLINE=1 a missing file fails as before."""
+    try:
+        return hf_hub_download(repo, filename, local_files_only=True)
+    except LocalEntryNotFoundError:
+        return hf_hub_download(repo, filename)
+
+
+# ORT keeps the high-water mark of its CPU memory arena for the life of the
+# session: the first search after load grew the process by +490 MB RSS /
+# +1,237 MB private (two 16-document reranker batches), and it never came
+# back. Shrinking the arena at the end of every run returns what a run
+# allocated; outputs are bit-identical and the per-batch time is within
+# noise (983-1160 vs 985-1079 ms; code audit 2026-09-21, perf-04b).
+_ARENA_SHRINK = ("memory.enable_memory_arena_shrinkage", "cpu:0")
+
+
+def make_run_options():
+    """The RunOptions every session.run() of the reranker and the embedder
+    passes: shrink the CPU arena after each run (see _ARENA_SHRINK)."""
+    options = ort.RunOptions()
+    options.add_run_config_entry(*_ARENA_SHRINK)
+    return options
+
+
+def make_ort_session(model_path):
+    """A CPU InferenceSession tuned for shared, small-vCPU hosts.
+
+    Two deviations from onnxruntime defaults: (1) spin-waiting between requests
+    is disabled -- by default ORT worker threads busy-spin after an inference
+    for a microsecond-scale latency win, which on the Space's shared 2 vCPUs
+    burns quota between queries; (2) the intra-op thread count is
+    env-configurable (CANLEX_ORT_THREADS) so concurrent sessions can be capped
+    below core count if contention is ever observed. Unset, ORT's default
+    (all cores) is kept, so single-query numerics and speed are unchanged."""
+    opts = ort.SessionOptions()
+    threads = int(os.environ.get("CANLEX_ORT_THREADS", "0"))
+    if threads:
+        opts.intra_op_num_threads = threads
+    opts.add_session_config_entry("session.intra_op.allow_spinning", "0")
+    opts.add_session_config_entry("session.inter_op.allow_spinning", "0")
+    return ort.InferenceSession(model_path, sess_options=opts,
+                                providers=["CPUExecutionProvider"])
+
+
+class Reranker:
+    """Cross-encoder that scores (query, section) pairs for relevance.
+
+    Loads a BGE reranker as ONNX and runs it on CPU -- no API key; the model is
+    downloaded once and cached. A cross-encoder reads the query and section
+    jointly, so it judges true relevance far better than the BM25/embedding
+    similarities used to build the candidate pool.
+    """
+
+    def __init__(self):
+        model_path = hub_file(RERANK_REPO, RERANK_ONNX)
+        tok_path = hub_file(RERANK_REPO, "tokenizer.json")
+        self.session = make_ort_session(model_path)
+        self.run_options = make_run_options()
+        self.input_names = {i.name for i in self.session.get_inputs()}
+
+        self.tokenizer = Tokenizer.from_file(tok_path)
+        # only_second keeps the query intact and truncates the (longer) section.
+        self.tokenizer.enable_truncation(max_length=MAX_TOKENS, strategy="only_second")
+        # An untruncated twin measures the query (see cap_query): a truncated
+        # tokenizer cannot encode a lone sequence under only_second.
+        self._measure = Tokenizer.from_file(tok_path)
+
+    @staticmethod
+    def cap_query(tokenizer, query, max_tokens=_MAX_QUERY_TOKENS):
+        """`query` cut to its first `max_tokens` tokens of `tokenizer` (an
+        untruncated tokenizer), on a token boundary of the original string;
+        the query itself, untouched, when it fits."""
+        enc = tokenizer.encode(query, add_special_tokens=False)
+        if len(enc.ids) <= max_tokens:
+            return query
+        return query[:enc.offsets[max_tokens - 1][1]]
+
+    def score(self, query, documents):
+        """Return a relevance logit for each document paired with the query.
+
+        Higher means more relevant. The returned list is aligned with `documents`.
+
+        Each (query, document) pair is scored in an ONNX run of its own -- a
+        contract, not a tuning choice. The int8 model quantizes activations
+        per run over the whole input tensor (DynamicQuantizeLinear), so in a
+        padded batch a pair's logit moved with its batch companions: all 192
+        of a 12 x 16 sample moved, by up to 8.8. One run per pair makes a
+        score a property of the pair alone -- the same in the rerank pool,
+        the pinpoint pass and a test -- and carries no padding:
+        search(top_k=6) p50 over 40 eval questions 1,425 -> 951 ms. Adopted
+        on the eval (audit follow-up 2026-09-21, group A item 8a): legacy
+        Hit@3 .9583 -> .9688, MRR .8651 -> .8668; holdout (undated) Hit@3
+        .9250 -> .9500, MRR .8383 -> .8404; no Hit@1 moved; top-6 order
+        moves on 209 of 339 questions.
+        """
+        if not documents:
+            return []
+        query = self.cap_query(self._measure, query)
+        encs = self.tokenizer.encode_batch(
+            [(query, doc[:_MAX_DOC_CHARS]) for doc in documents])
+        return [self._logit(enc) for enc in encs]
+
+    def _logit(self, enc):
+        """The logit of one encoded (query, document) pair, run alone."""
+        feed = {"input_ids": np.asarray([enc.ids], dtype=np.int64),
+                "attention_mask": np.asarray([enc.attention_mask], dtype=np.int64)}
+        if "token_type_ids" in self.input_names:
+            feed["token_type_ids"] = np.asarray([enc.type_ids], dtype=np.int64)
+        logits = self.session.run(None, feed, self.run_options)[0]
+        return float(np.asarray(logits, dtype=np.float32).reshape(-1)[0])
+
+
+def main():
+    import time
+    print(f"Loading {RERANK_REPO} ({RERANK_ONNX}) ...")
+    reranker = Reranker()
+    query = "powers of arrest without warrant"
+    docs = [
+        "Arrest without warrant. A peace officer may arrest without warrant a "
+        "person who has committed a criminal offence.",
+        "Definitions. In this Act, fish means any fish and includes shellfish, "
+        "crustaceans and marine animals.",
+        "Importation. It is prohibited to import cannabis except as authorized "
+        "under this Act.",
+    ]
+    start = time.perf_counter()
+    scores = reranker.score(query, docs)
+    elapsed = (time.perf_counter() - start) * 1000
+    print(f"\nQuery: {query!r}")
+    for doc, score in sorted(zip(docs, scores), key=lambda x: x[1], reverse=True):
+        print(f"  {score:8.3f}  {doc[:62]}")
+    print(f"\n{elapsed:.0f} ms for {len(docs)} documents")
+
+
+if __name__ == "__main__":
+    main()

canlex/sweep.py

@@ -0,0 +1,141 @@
+"""Coordinate-descent tuning sweep for the four retrieval knobs.
+
+Runs the 141-question eval repeatedly, varying one knob at a time while holding
+the others at their current best. Picks the value that maximises Hit@5 (with
+MRR as the tiebreak) and continues to the next knob. One pass through all four
+knobs is usually enough to settle.
+
+The knobs are read from env vars at index-load time (see canlex/index.py), so
+each combination is exercised in a fresh subprocess of canlex.eval. Outputs go
+to data/eval/sweep.log and a compact JSON summary to data/eval/sweep.json.
+
+    py -m canlex.sweep
+"""
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+LOG = ROOT / "data" / "eval" / "sweep.log"
+SUMMARY = ROOT / "data" / "eval" / "sweep.json"
+
+# (env var, list of candidate values, current default). The defaults match the
+# literals in canlex/index.py; values bracket each on a roughly geometric grid.
+KNOBS = [
+    ("CANLEX_MN_WEIGHT",         [0.0012, 0.0024, 0.005, 0.01],   0.0024),
+    ("CANLEX_MN_CAP",            [0.012, 0.024, 0.05, 0.1],       0.024),
+    ("CANLEX_REG_PENALTY",       [0.002, 0.004, 0.008, 0.016],    0.004),
+    ("CANLEX_BACKMATTER_PENALTY",[0.002, 0.004, 0.008, 0.016],    0.004),
+    # The hierarchy prior's two guidance tiers (roadmap T5.4): official
+    # guidance, and CanLex's own curated commentary below it. Swept together
+    # because they trade against each other on the same questions.
+    ("CANLEX_MEMO_PENALTY",      [0.006, 0.008, 0.010],           0.008),
+    ("CANLEX_COMMENTARY_PENALTY",[0.0, 0.002, 0.004, 0.006],      0.0),
+    # IRCC's applicant-facing pages, below official guidance (T5.4; the
+    # IRCC 'Overcome criminal convictions' page outranked IRPR s. 18 on a
+    # legacy rehabilitation question by 0.0135).
+    ("CANLEX_APPLICANT_PENALTY", [0.0, 0.008, 0.014, 0.02],       0.0),
+]
+
+
+_METRIC_RE = re.compile(
+    r"Hit@1:\s*([\d.]+).*?Hit@3:\s*([\d.]+).*?Hit@5:\s*([\d.]+).*?"
+    r"Hit@10:\s*([\d.]+).*?MRR:\s*([\d.]+)", re.S)
+
+
+def _run_eval(env_overrides: dict[str, float]) -> dict[str, float]:
+    """Run canlex.eval once with the given env overrides; return metrics dict."""
+    env = dict(os.environ)
+    env["CANLEX_EVAL_SWEEP"] = "1"   # holdout questions stay sweep-blind
+    for k, v in env_overrides.items():
+        env[k] = f"{v}"
+    proc = subprocess.run(
+        [sys.executable, "-u", "-m", "canlex.eval"],
+        capture_output=True, text=True, env=env, cwd=ROOT,
+    )
+    if proc.returncode != 0:
+        raise RuntimeError(f"eval failed (exit {proc.returncode}):\n"
+                           f"{proc.stderr[-800:]}")
+    m = _METRIC_RE.search(proc.stdout)
+    if not m:
+        raise RuntimeError(f"could not parse eval output:\n{proc.stdout[-800:]}")
+    h1, h3, h5, h10, mrr = (float(x) for x in m.groups())
+    n_misses = proc.stdout.count("miss(es)")          # 0 if we end up at 100
+    return {"hit1": h1, "hit3": h3, "hit5": h5, "hit10": h10, "mrr": mrr,
+            "stdout": proc.stdout}
+
+
+def _score(metrics: dict[str, float]) -> tuple[float, float]:
+    """Order by Hit@5 then MRR (both higher is better)."""
+    return (metrics["hit5"], metrics["mrr"])
+
+
+def main():
+    LOG.parent.mkdir(parents=True, exist_ok=True)
+    log = LOG.open("w", encoding="utf-8")
+    log.write(f"# CanLex tuning sweep -- {time.strftime('%Y-%m-%d %H:%M:%S')}\n")
+    print(f"# CanLex tuning sweep -- writing log to {LOG}\n")
+
+    current = {name: default for name, _values, default in KNOBS}
+    all_runs: list[dict] = []
+
+    # Baseline at current defaults.
+    print("Baseline:", current)
+    log.write(f"\nBaseline: {current}\n")
+    baseline = _run_eval(current)
+    print(f"  Hit@5={baseline['hit5']:.3f}  MRR={baseline['mrr']:.3f}")
+    log.write(f"  Hit@5={baseline['hit5']:.3f}  MRR={baseline['mrr']:.3f}\n")
+    all_runs.append({"values": dict(current), "metrics":
+                     {k: baseline[k] for k in ("hit1", "hit3", "hit5", "hit10", "mrr")}})
+    best_metrics = baseline
+
+    for name, values, _default in KNOBS:
+        print(f"\nSweeping {name} in {values} (others held at {current})...")
+        log.write(f"\nSweeping {name} in {values} (others held at {current})\n")
+        local_best = (current[name], _score(best_metrics), best_metrics)
+        for v in values:
+            if v == current[name]:
+                # Re-use the already-measured baseline at this knob value.
+                metrics = best_metrics
+            else:
+                run_values = dict(current, **{name: v})
+                metrics = _run_eval(run_values)
+            row = (f"  {name}={v!r:<8s} -> Hit@1={metrics['hit1']:.3f} "
+                   f"Hit@3={metrics['hit3']:.3f} Hit@5={metrics['hit5']:.3f} "
+                   f"Hit@10={metrics['hit10']:.3f} MRR={metrics['mrr']:.3f}")
+            print(row); log.write(row + "\n"); log.flush()
+            all_runs.append({"values": dict(current, **{name: v}),
+                             "metrics": {k: metrics[k] for k in
+                                         ("hit1", "hit3", "hit5", "hit10", "mrr")}})
+            score = _score(metrics)
+            if score > local_best[1]:
+                local_best = (v, score, metrics)
+        if local_best[0] != current[name]:
+            print(f"  ! {name}: {current[name]} -> {local_best[0]}  "
+                  f"(Hit@5 {best_metrics['hit5']:.3f} -> {local_best[2]['hit5']:.3f})")
+            log.write(f"  ! {name}: {current[name]} -> {local_best[0]}\n")
+        current[name] = local_best[0]
+        best_metrics = local_best[2]
+
+    print(f"\nBest: {current}")
+    print(f"  Hit@1={best_metrics['hit1']:.3f}  Hit@3={best_metrics['hit3']:.3f}  "
+          f"Hit@5={best_metrics['hit5']:.3f}  Hit@10={best_metrics['hit10']:.3f}  "
+          f"MRR={best_metrics['mrr']:.3f}")
+    log.write(f"\nBest: {current}\n  Hit@1={best_metrics['hit1']:.3f} "
+              f"Hit@5={best_metrics['hit5']:.3f}  MRR={best_metrics['mrr']:.3f}\n")
+    log.close()
+    SUMMARY.write_text(json.dumps({
+        "best": current,
+        "best_metrics": {k: best_metrics[k] for k in
+                         ("hit1", "hit3", "hit5", "hit10", "mrr")},
+        "runs": all_runs,
+    }, indent=2), encoding="utf-8")
+    print(f"\nLog: {LOG}\nSummary: {SUMMARY}")
+
+
+if __name__ == "__main__":
+    main()

canlex/synonyms.py

@@ -0,0 +1,208 @@
+"""Query-side expansion of legal abbreviations and informal terms.
+
+Statutes use formal wording -- "application for protection", "removal order" --
+but users (and a model drafting a search) reach for everyday shorthand: "PRRA",
+"H&C", "deportation". Before retrieval, expand_query() appends the canonical
+statutory terms for any abbreviation or nickname it recognises, so the BM25 and
+semantic stages can match the provision's actual language. It only ever ADDS
+words -- the user's own phrasing is left untouched -- and the cross-encoder
+reranker still sees the original query, so precision is unaffected.
+
+    python -m canlex.synonyms "PRRA eligibility and an H&C application"
+"""
+import re
+import sys
+
+# (trigger, canonical terms to append). The trigger is a regex fragment matched
+# case-insensitively as a whole word. Keep this list high-precision: an entry
+# earns its place only when the shorthand is unambiguous in Canadian border,
+# immigration, customs, financial-crime or labour law.
+_SYNONYMS = [
+    # Immigration and refugee law
+    (r"prra", "pre-removal risk assessment application for protection"),
+    (r"pre[- ]removal risk assessment", "application for protection"),
+    (r"h\s*&\s*c", "humanitarian and compassionate"),
+    (r"rad", "refugee appeal division"),
+    (r"rpd", "refugee protection division"),
+    (r"iad", "immigration appeal division"),
+    (r"irb", "immigration and refugee board"),
+    (r"trp", "temporary resident permit"),
+    (r"deportation", "removal order"),
+    # "No right to appeal (a removal order)" is how users ask about IRPA
+    # s. 64, but the section itself never says "removal order" or "right" --
+    # its vocabulary is "no appeal may be made ... inadmissible on grounds
+    # of ... serious criminality". Bridge the gap.
+    (r"no (?:right (?:to|of) )?appeal",
+     "no appeal may be made inadmissibility serious criminality"),
+    # "Standard of proof" is how users ask about IRPA s. 33, whose own words
+    # are "reasonable grounds to believe" that the facts "have occurred".
+    (r"standard of proof", "reasonable grounds to believe facts constitute"),
+    # "Apply for (criminal) rehabilitation" points at IRPR s. 17, a tiny
+    # section titled only "Prescribed period" whose vocabulary is the
+    # five-year period after "completion of an imposed sentence".
+    (r"appl(?:y|ication) for (?:criminal )?rehabilitation",
+     "prescribed period five years completion of imposed sentence"),
+    (r"misrep", "misrepresentation"),
+    (r"ircc", "immigration refugees and citizenship canada"),
+    (r"eta", "electronic travel authorization"),
+    (r"trv", "temporary resident visa"),
+    (r"arc", "authorization to return to canada removal order"),
+    (r"danger opinion", "danger to the public refoulement protection"),
+    (r"ministerial relief", "national interest exception inadmissibility"),
+    (r"poe", "port of entry"),
+    # French for the IRB Chairperson's Guidelines, which exist in English only
+    (r"(?:directives?|lignes? directrices?) (?:de la|du) (?:cisr|président)",
+     "Chairperson's Guideline Immigration and Refugee Board"),
+    # Language of service: the Official Languages Act never says 'French'
+    # or 'English' in Part IV -- its words are 'either official language',
+    # 'communicate with and obtain available services', 'significant demand'.
+    (r"(?:served|service|serve|communicate) (?:in|en) (?:french|english|français|anglais)",
+     "communicate with and obtain available services in either official language significant demand"),
+    # Device examinations: officers say 'cell phone' / 'laptop'; Customs Act
+    # s. 99 says 'goods' and ENF 12 s. 6.15 'digital devices'. Without the
+    # bridge an arrest-context SCC case (Fearon) and two unrelated 'without
+    # warrant' marginal notes outrank the governing border authorities.
+    (r"(?:cell ?phones?|smart ?phones?|mobile (?:phones?|devices?)|laptops?"
+     r"|tablets?|digital devices?|electronic devices?)",
+     "examination of goods digital device electronic device"),
+    # Money: 'cash', 'USD', '10k' never reach PCMLTFA s. 12's own words
+    # ('currency or monetary instruments ... prescribed amount').
+    (r"(?:cash|us ?dollars|usd|10 ?k|ten thousand dollars)",
+     "currency or monetary instruments report prescribed amount"),
+    (r"flagpol(?:e|ing)", "leaving canada and reentering port of entry"),
+    # Tax evasion: 'evasion' stems to 'evas' and the Income Tax Act's own
+    # words ('evaded or attempted to evade ... payment of taxes') stem to
+    # 'evad', and s. 239 is titled only 'Other offences and punishment' -- so
+    # without the bridge "penalty for tax evasion" never reaches s. 239.
+    # 'tax' is required in the trigger, so customs-duty evasion is untouched.
+    (r"(?:tax(?:es)? evasion|evasion of (?:income )?tax(?:es)?"
+     r"|evad(?:e|ed|ing) (?:income )?tax(?:es)?|tax fraud"
+     r"|[ée]vasion fiscale|fraude fiscale)",
+     "evaded or attempted to evade payment of taxes false or deceptive "
+     "statements in a return offences and punishment"),
+    # The single most common US-screening term: bridge DUI shorthand to the
+    # Criminal Code s. 320.14 vocabulary the equivalency analysis turns on.
+    (r"(?:dwi|dui|drunk driving)",
+     "operation while impaired blood alcohol concentration conveyance"),
+    # Obstructing or assaulting an officer. The everyday phrasing matches the
+    # generic obstruction sections of the Quarantine and Excise Acts word for
+    # word ('hinder or wilfully obstruct ... duties or functions'), while the
+    # four provisions an officer actually needs each speak differently:
+    # Customs Act s. 153.1 'interfere with or molest', IRPA s. 129(1)(d)
+    # 'obstructs or impedes', Criminal Code s. 129 'resists or wilfully
+    # obstructs' and s. 270 'assaults a public officer or peace officer'.
+    # The trigger requires the conduct word to GOVERN 'officer' -- at most two
+    # words in between -- so 'what factors impede an immigration officer' and
+    # 'grab the officer's business card' are untouched. It also takes the
+    # plain words an officer writes a fact pattern in ('shoved', 'spat at'),
+    # none of which appears in any of the four provisions. Verbs that are
+    # common in ordinary legal prose ('impede', 'threaten', 'push', 'grab')
+    # are deliberately left out: they false-fire more often than they help.
+    (r"(?:obstruct|hinder|resist|assault|shov|punch|strike|struck"
+     r"|kick|spit|spat|swung|swing)\w*"
+     r"(?:\W+\w+){0,2}\W+(?:border services |peace |public |customs )?officers?",
+     "interfere with or molest an officer hindering an officer "
+     "obstructs or impedes an officer in the performance of the officer's "
+     "duties offences relating to officers "
+     "resists or wilfully obstructs a public officer or peace officer in the "
+     "execution of his duty "
+     "assaults a public officer or peace officer engaged in the execution of "
+     "his duty"),
+    # Arrest at the border. Customs Act s. 163.5 is what lends a designated
+    # officer the peace officer's arrest powers, but it never says 'arrest
+    # without warrant' -- its words are 'the powers and obligations of a peace
+    # officer under sections 495 and 497'. 'Fresh pursuit' is the officer's
+    # term for Criminal Code s. 494(1)(b)(ii)'s 'freshly pursued'.
+    (r"(?:arrest(?:s|ed|ing)? without (?:a )?warrant"
+     r"|without (?:a )?warrant arrest\w*"
+     r"|(?:found|finds|find) committing"
+     r"|fresh(?:ly)? pursu\w+|continuous pursuit)",
+     "arrest without warrant finds committing a criminal offence escaping "
+     "from and freshly pursued powers and obligations of a peace officer "
+     "under sections 495 and 497 designated officer at a customs office"),
+    (r"personal exemption",
+     "returning persons exemption duty free allowance"),
+    # Health-plan vocabulary. Drug brand and molecule names appear nowhere in
+    # the PSHCP documents: the rule that answers a Botox question is the
+    # exclusion of prescription drugs 'used for cosmetic purposes'.
+    (r"(?:botox|botulinum(?: toxin)?|cosmetic (?:drugs?|purposes?|treatments?"
+     r"|procedures?|surgery))",
+     "prescription drugs used for cosmetic purposes exclusions expenses for "
+     "services or products for cosmetic purposes only prior authorization"),
+    # Likewise the plans never name a provincial insurer: RAMQ, OHIP and the
+    # rest are all 'a provincial/territorial health insurance plan' that the
+    # PSHCP supplements, with a co-ordination rule for a Quebec resident's
+    # prescription drug claim. Only the insurers' own names trigger this:
+    # 'provincial health plan' by itself is ordinary immigration vocabulary
+    # (a permit holder's provincial coverage) and must not reach the PSHCP.
+    (r"(?:ramq|ohip|r[ée]gie de l'assurance[- ]maladie(?: du qu[ée]bec)?"
+     r"|qu[ée]bec (?:drug|prescription drug) (?:insurance|plan))",
+     "provincial/territorial health insurance plan supplementary coverage "
+     "co-ordination of benefits Quebec resident submitting a prescription "
+     "drug claim"),
+    (r"(?:nexus|trusted traveller)",
+     "presentation of persons alternative manner authorization"),
+    # Border and customs
+    (r"cbsa", "canada border services agency"),
+    (r"bsos?", "border services officer"),
+    (r"amps", "administrative monetary penalty system"),
+    # Financial-crime and labour
+    (r"fintrac", "financial transactions and reports analysis centre"),
+    (r"njc", "national joint council"),
+    # French -> English bridges. The corpus is English-only, but officers work
+    # bilingually and drafting agents emit French terms; with the tokenizer's
+    # accent folding these triggers match either accented or folded forms.
+    # High-precision statutory anchors only.
+    (r"erar", "pre-removal risk assessment application for protection"),
+    (r"asfc", "canada border services agency"),
+    (r"interdiction de territoire", "inadmissibility inadmissible"),
+    (r"grande criminalit[ée]", "serious criminality"),
+    (r"criminalit[ée]", "criminality convicted offence"),
+    (r"renvoi", "removal order"),
+    (r"saisie?s?", "seizure seized goods"),
+    # 'devises' is the everyday word for currency; the PCMLTFA's French text
+    # says 'espèces et effets' -- bridge to both statutory vocabularies.
+    (r"devises", "currency monetary instruments espèces effets"),
+    (r"r[ée]sidents? permanents?", "permanent resident"),
+    (r"[ée]trangers?", "foreign national"),
+    (r"d[ée]tention", "detention detained review"),
+    (r"r[ée]fugi[ée]s?", "refugee protection convention"),
+    (r"douanes?", "customs duties"),
+    (r"marchandises?", "goods imported"),
+    (r"d[ée]claration", "report declaration"),
+    (r"armes? [àa] feu", "firearm weapon"),
+    (r"contr[ôo]le", "examination search"),
+    (r"agents? des services frontaliers", "border services officer"),
+    (r"permis de travail", "work permit"),
+    (r"permis d'[ée]tudes", "study permit"),
+    (r"visa de r[ée]sident temporaire", "temporary resident visa"),
+    (r"r[ée]adaptation", "rehabilitation prescribed period"),
+    (r"expulsion", "deportation removal order"),
+    (r"passeurs?", "human smuggling organized entry"),
+    (r"fouille", "search of the person strip search"),
+    (r"griefs?", "grievance adjudication"),
+    (r"convention collective", "collective agreement"),
+    (r"heures suppl[ée]mentaires", "overtime hours of work"),
+]
+
+_COMPILED = [(re.compile(rf"\b{trigger}\b", re.IGNORECASE), expansion)
+             for trigger, expansion in _SYNONYMS]
+
+
+def expand_query(query):
+    """Return `query` with canonical statutory terms appended for every legal
+    abbreviation it contains; return it unchanged if it contains none."""
+    additions = [exp for pattern, exp in _COMPILED if pattern.search(query)]
+    if not additions:
+        return query
+    return f"{query} {' '.join(additions)}"
+
+
+def main():
+    query = " ".join(sys.argv[1:]) or "PRRA eligibility and an H&C application"
+    print(f"query:    {query}")
+    print(f"expanded: {expand_query(query)}")
+
+
+if __name__ == "__main__":
+    main()

canlex/tariff_schedule.py

@@ -0,0 +1,287 @@
+"""Ingest Chapters 98 and 99 of the Customs Tariff Schedule.
+
+The Customs Tariff's Schedule is the Harmonized System classification of goods
+-- chapters 1-97 categorise every imported good for duty purposes. Those 97
+chapters are huge and outside CanLex's scope, but **chapters 98 and 99 are
+different**: they carry Canada's "special classification" provisions, which
+matter for almost every CBSA border interaction:
+
+  - Chapter 98 (non-commercial): traveller exemptions, settler's effects,
+    Canadian goods returned, ancestral household effects, conveyances
+    temporarily imported by a resident, etc.
+  - Chapter 99 (commercial): temporary importations, end-use programs,
+    government imports, reduced-rate goods for specific industries.
+
+Source: the CBSA's HTML edition of the current Customs Tariff. The Justice
+Laws XML for the Act (C-54.011) does NOT include the Schedule.
+
+Chunking is one chunk per 4-digit HEADING (98.01, 98.02, ...) plus one chunk
+per chapter for its Notes and Subheading Notes -- a heading is the natural
+unit of legal classification (the eight- and ten-digit items below it are the
+same rule with finer rate granularity).
+
+    py -m canlex.tariff_schedule [--allow-shrink]
+"""
+import re
+import sys
+from collections import defaultdict
+
+from bs4 import BeautifulSoup
+
+from ._common import (fetch_cached, norm_ws as _norm, preserve_dropped,
+                      stored_chunks, write_corpus)
+from .config import PROCESSED_DIR, RAW_DIR
+
+RAW = RAW_DIR / "tariff_schedule"
+OUT = PROCESSED_DIR / "tariff_schedule.json"
+
+# The 2026 edition is the current Customs Tariff at the time of writing. To
+# refresh: bump the year in the URL and `edition` once CBSA publishes the next.
+SOURCES = {
+    "ch98": {
+        "code": "ch98",
+        "chapter": "98",
+        "title": "Special classification provisions — non-commercial",
+        "url": ("https://www.cbsa-asfc.gc.ca/trade-commerce/tariff-tarif/"
+                "2026/html/00/ch98-eng.html"),
+        "edition": "2026",
+    },
+    "ch99": {
+        "code": "ch99",
+        "chapter": "99",
+        "title": "Special classification provisions — commercial",
+        "url": ("https://www.cbsa-asfc.gc.ca/trade-commerce/tariff-tarif/"
+                "2026/html/00/ch99-eng.html"),
+        "edition": "2026",
+    },
+}
+
+
+def _fetch(url, dest, force=False):
+    return fetch_cached(url, dest, force=force)
+
+
+def _heading_of(item):
+    """Map a tariff-item code to its 4-digit heading. The schedule uses three
+    levels:
+      - '98.01'       (4 digits)           -- heading
+      - '9801.10'     (6 digits)           -- subheading
+      - '9801.10.10'  (8 digits)           -- tariff item
+    All three roll up to heading '98.01'."""
+    digits = re.sub(r"\D", "", item)
+    if len(digits) < 4:
+        return None
+    return f"{digits[:2]}.{digits[2:4]}"
+
+
+def _chapter_part(chapter):
+    """The `part` every chunk of a chapter carries -- and, because nothing else
+    on a chunk names its chapter, the per-chapter identity preserve_failed
+    matches on. Derived in one place so the two stay in step."""
+    return f"Schedule, Chapter {chapter}"
+
+
+def _collect_notes(main):
+    """Return the chapter's Notes + Subheading Notes as a single text block."""
+    out = []
+    for label in ("Notes", "Subheading Notes"):
+        h2 = main.find(
+            "h2",
+            string=lambda s, lbl=label: s and lbl in s and (
+                lbl != "Notes" or "Subheading" not in s))
+        if not h2:
+            continue
+        parts = []
+        sib = h2.find_next_sibling()
+        while sib and sib.name != "h2":
+            t = _norm(sib.get_text(" ", strip=True))
+            if t:
+                parts.append(t)
+            sib = sib.find_next_sibling()
+        if parts:
+            out.append(f"{label}:\n" + "\n".join(parts))
+    return "\n\n".join(out)
+
+
+def parse_chapter(html, src):
+    """Parse one Customs Tariff Schedule chapter into chunks."""
+    soup = BeautifulSoup(html, "html.parser")
+    main = soup.find("main")
+    if main is None:
+        return []
+    for sup in main.find_all("sup"):
+        sup.decompose()
+
+    chunks = []
+    chapter = src["chapter"]
+    citation_root = f"Customs Tariff, Sched., Ch. {chapter}"
+
+    # Chapter Notes + Subheading Notes -- one chunk per chapter.
+    notes_body = _collect_notes(main)
+    if notes_body:
+        chunks.append({
+            "id": f"tariff-sched-ch{chapter}-notes",
+            "doc_type": "legislation",
+            "act_code": "C-54.011",
+            "act_short": "Customs Tariff",
+            "act_name": "Customs Tariff",
+            "section": f"Sch-Ch{chapter}-Notes",
+            "marginal_note": (f"Chapter {chapter} Notes — "
+                              f"{src['title']}"),
+            "part": _chapter_part(chapter),
+            "division": "",
+            "heading": src["title"],
+            "text": notes_body,
+            "history": "",
+            "last_amended": "",
+            "current_to": src["edition"],
+            "citation": f"{citation_root}, Notes",
+            "source_url": src["url"],
+        })
+
+    # Walk every row in the schedule table, grouping by 4-digit heading.
+    table = main.find("table")
+    if table is None:
+        return chunks
+
+    rows_by_heading = defaultdict(list)   # heading -> list of (item, ss, desc, unit, mfn, pref)
+    heading_desc = {}                     # heading -> the 4-digit row's description
+
+    for tr in table.find_all("tr"):
+        cells = tr.find_all(["td", "th"], recursive=False)
+        if not cells:
+            continue
+        first = _norm(cells[0].get_text(" ", strip=True))
+        if not first or first == "Tariff Item":
+            continue                       # header row or blank
+        heading = _heading_of(first)
+        if heading is None:
+            continue
+
+        def col(i):
+            return _norm(cells[i].get_text(" ", strip=True)) if i < len(cells) else ""
+
+        ss, desc, unit, mfn, pref = col(1), col(2), col(3), col(4), col(5)
+
+        # A 4-digit row carries only the heading number and description -- store
+        # it once. Otherwise it's a subheading/item row; remember its rate cells.
+        digits = re.sub(r"\D", "", first)
+        if len(digits) == 4 and desc and heading not in heading_desc:
+            heading_desc[heading] = desc
+        rows_by_heading[heading].append((first, ss, desc, unit, mfn, pref))
+
+    for heading in sorted(rows_by_heading):
+        desc = heading_desc.get(heading, "")
+        rows = rows_by_heading[heading]
+        if not desc:
+            # No bare 4-digit row -- fall back to the first row's description.
+            desc = next((r[2] for r in rows if r[2]), "")
+
+        # Format the chunk: heading description first, then the items as a
+        # readable list. Suppress description on item/subheading rows that just
+        # repeat the heading desc verbatim (very common in this Schedule).
+        lines = []
+        for item, ss, item_desc, unit, mfn, pref in rows:
+            d = re.sub(r"\D", "", item)
+            if len(d) == 4:
+                continue                   # the heading row, already in desc
+            label = f"  {item}"
+            if ss:
+                label += f" (SS {ss})"
+            extras = []
+            if item_desc and item_desc != desc:
+                extras.append(item_desc)
+            rate = []
+            if mfn:
+                rate.append(f"MFN {mfn}")
+            if pref:
+                rate.append(f"Pref: {pref}")
+            if unit and unit != "-":
+                rate.append(f"Unit {unit}")
+            if rate:
+                extras.append(" | ".join(rate))
+            tail = " — " + "; ".join(extras) if extras else ""
+            lines.append(label + tail)
+
+        body = f"Heading {heading} — {desc}"
+        if lines:
+            body += "\n\nTariff items:\n" + "\n".join(lines)
+
+        chunks.append({
+            "id": f"tariff-sched-{heading.replace('.', '-')}",
+            "doc_type": "legislation",
+            "act_code": "C-54.011",
+            "act_short": "Customs Tariff",
+            "act_name": "Customs Tariff",
+            "section": f"Sch-{heading}",
+            "marginal_note": desc[:200],
+            "part": _chapter_part(chapter),
+            "division": "",
+            "heading": src["title"],
+            "text": body,
+            "history": "",
+            "last_amended": "",
+            "current_to": src["edition"],
+            "citation": f"Customs Tariff, Sched., heading {heading}",
+            "source_url": src["url"],
+        })
+
+    return chunks
+
+
+def preserve_failed(chunks, failures, stored):
+    """Re-attach the last-good chunks of chapters this run could not rebuild.
+
+    Only two chapters make up this corpus, so losing one is losing a third to
+    two thirds of it -- and a CBSA page that has been reshaped (or a fetch that
+    died mid-run) is indistinguishable from a chapter that legitimately
+    vanished until a human looks. Serving last week's copy of ch. 98 beats
+    serving no traveller exemptions at all. Restricted to the chapters that
+    actually failed. Returns (chunks, preserved chapter identities).
+    """
+    failed = {_chapter_part(chapter) for chapter, _why in failures}
+    return preserve_dropped(chunks, stored, key=lambda c: c["part"],
+                            only=failed)
+
+
+def build(allow_shrink=False, force=False):
+    all_chunks, failures = [], []
+    for src in SOURCES.values():
+        print(f"Ingesting Customs Tariff Schedule {src['code']} ...")
+        try:
+            html = _fetch(src["url"], RAW / f"{src['code']}.html",
+                          force=force)
+            chunks = parse_chapter(html, src)
+        except Exception as exc:
+            print(f"  !! {src['code']}: {type(exc).__name__}: {exc}")
+            failures.append((src["chapter"], f"{type(exc).__name__}: {exc}"))
+            continue
+        if not chunks:
+            # A chapter never parses to nothing legitimately -- an empty result
+            # means the page moved or changed shape, so treat it as a failure
+            # rather than letting it silently shrink the corpus.
+            print(f"  !! {src['code']}: no chunks parsed")
+            failures.append((src["chapter"], "no chunks parsed"))
+            continue
+        all_chunks.extend(chunks)
+        print(f"  {len(chunks)} chunks")
+
+    all_chunks, preserved = preserve_failed(all_chunks, failures,
+                                            stored_chunks(OUT))
+    if preserved:
+        print(f"  kept the last-good copy of {len(preserved)} chapter(s): "
+              f"{', '.join(preserved)}")
+    if not write_corpus(OUT, all_chunks, indent=1, allow_shrink=allow_shrink):
+        return False
+    print(f"\n{len(all_chunks)} tariff-schedule chunks from "
+          f"{len(SOURCES) - len(failures)} chapter(s) -> {OUT}")
+    return True
+
+
+def main():
+    sys.exit(0 if build(allow_shrink="--allow-shrink" in sys.argv,
+                        force="--force" in sys.argv) else 1)
+
+
+if __name__ == "__main__":
+    main()

canlex/transborder.py

@@ -0,0 +1,778 @@
+"""Transborder criminality: IRPA s. 36(2.1), IRPR ss. 19, 228(1)(a.01) and
+229(1)(d.1) (roadmap T2.1, with the critic's correction).
+
+s. 36(2)(d) is repealed (2023, c. 32, s. 53); s. 36(2.1) now reads 'A foreign
+national is inadmissible on grounds of transborder criminality for
+committing, on entering Canada, a prescribed offence under an Act of
+Parliament', and IRPR s. 19(1) prescribes the offences:
+
+  (a) any indictable offence under the Criminal Code;
+  (b) any indictable offence under the Immigration and Refugee Protection Act;
+  (c)-(g) listed offences under the Firearms Act, the Customs Act (by goods),
+      the Controlled Drugs and Substances Act and the Cannabis Act.
+
+(a) and (b) are OPEN CLASSES, not lists, so they are rules here, driven by the
+offence's mode -- hybrid offences are deemed indictable by s. 19(2). Only
+(c)-(g) are tabulated, and that table is curated legal content
+(data/curated/transborder.json, HELD FOR REVIEW). A composed release ships
+without it: (a)/(b) still work and the answer says the (c)-(g) table awaits
+review.
+
+Also statute-mechanical, and transcribed here because the removal path turns
+on them: which offences the Minister's delegate may decide by a deportation
+order (s. 228(1)(a.01)(i)-(v)) and which go to the Immigration Division
+(s. 229(1)(d.1)); the s. 36(3)(c) rule that rehabilitation does not reach
+s. 36(2.1); and the s. 36(3)(e)(i) contraventions carve-out, checked against
+the Contraventions Regulations (SOR-96-313) in the corpus.
+
+Every statutory phrase below is pinned against the corpus text by
+tests/test_transborder.py. Every definition an officer reads is registered
+in canlex/reviewed_code.py.
+"""
+import json
+import re
+
+TABLE_FILE = "transborder.json"
+
+# ---- the Acts IRPR s. 19(1) names ------------------------------------------------
+# Corpus act code -> the Act's name as s. 19(1) writes it, the paragraph, and
+# whether the paragraph is an open class (a rule on the mode) or a list (the
+# curated table). Any other Act: s. 19(1) prescribes no offence under it.
+R19_ACTS = {
+    "C-46": {"name": "Criminal Code", "paragraph": "19(1)(a)", "open": True,
+             "aliases": ("criminal code", "cc", "ccc", "c-46", "code criminel")},
+    "I-2.5": {"name": "Immigration and Refugee Protection Act", "paragraph": "19(1)(b)",
+              "open": True,
+              "aliases": ("immigration and refugee protection act", "irpa", "i-2.5", "lipr")},
+    "F-11.6": {"name": "Firearms Act", "paragraph": "19(1)(c)", "open": False,
+               "aliases": ("firearms act", "f-11.6", "loi sur les armes a feu")},
+    "C-52.6": {"name": "Customs Act", "paragraph": "19(1)(d) and (e)", "open": False,
+               "aliases": ("customs act", "c-52.6", "loi sur les douanes")},
+    "C-38.8": {"name": "Controlled Drugs and Substances Act", "paragraph": "19(1)(f)",
+               "open": False,
+               "aliases": ("controlled drugs and substances act", "cdsa", "c-38.8", "lrcdas")},
+    "C-24.5": {"name": "Cannabis Act", "paragraph": "19(1)(g)", "open": False,
+               "aliases": ("cannabis act", "c-24.5", "loi sur le cannabis")},
+}
+
+# The goods classes IRPR s. 19(1)(d) and (e) turn on, as the Regulations
+# define them. The keys are CanLex's; the labels quote s. 19(1).
+GOODS = {
+    "child-pornography": "child pornography, as defined in subsection 163.1(1) of the "
+                         "Criminal Code",
+    "firearm-or-weapon": "an automatic firearm, imitation firearm, non-restricted firearm, "
+                         "prohibited ammunition, prohibited device, prohibited firearm, "
+                         "prohibited weapon, replica firearm, restricted firearm or "
+                         "restricted weapon, as those terms are defined in subsection "
+                         "84(1) of the Criminal Code",
+    "controlled-substance": "a substance listed in any of Schedules I to VI of the "
+                            "Controlled Drugs and Substances Act",
+    "schedule-ix-device": "a device listed in Schedule IX of that Act",
+    "cannabis": "cannabis, as defined in subsection 2(1) of the Cannabis Act",
+    "other": "goods of any other kind",
+}
+
+MODES = ("indictable", "hybrid", "summary-only")
+
+# IRPR s. 228(1)(a.01): the transborder offences the Minister's delegate
+# decides with a deportation order (the report 'shall not be referred to the
+# Immigration Division'). (subparagraph, act, provisions, goods condition).
+# For the Customs Act, '12' and '156' name the provision contravened: the
+# offence is under s. 160(1) 'that is ... a contravention of section 12' (or
+# 156). Every other prescribed offence is decided by the Immigration
+# Division: s. 229(1)(d.1).
+MD_ORDERABLE = (
+    ("228(1)(a.01)(i)", "C-46", ("86", "87", "90", "91", "95", "104", "320.15(1)"), None),
+    ("228(1)(a.01)(ii)", "I-2.5", ("124(1)(b)",), None),
+    ("228(1)(a.01)(iii)", "F-11.6", ("110",), None),
+    ("228(1)(a.01)(iv)", "C-52.6", ("159(1)",), "firearm-or-weapon"),
+    ("228(1)(a.01)(v)(A)", "C-52.6", ("12",), "firearm-or-weapon"),
+    ("228(1)(a.01)(v)(B)", "C-52.6", ("156",), None),
+)
+
+# ---- statute text the memo quotes (pinned in tests/test_transborder.py) ------------
+S36_2_1 = ("A foreign national is inadmissible on grounds of transborder criminality "
+           "for committing, on entering Canada, a prescribed offence under an Act of "
+           "Parliament")
+ALLEGATION_2_1 = "committing, on entering Canada, a prescribed offence under an Act of Parliament"
+R19_2 = ("an offence that may be prosecuted either summarily or by way of indictment is "
+         "deemed to be an offence punishable by way of indictment, even if it has been "
+         "prosecuted summarily")
+S36_3_C = ("the matters referred to in paragraphs (1)(b) and (c) and (2)(b) and (c) do "
+           "not constitute inadmissibility")
+S36_3_E = "inadmissibility under subsections (1) to (2.1) may not be based on an offence"
+S36_3_E_I = "designated as a contravention under the Contraventions Act"
+S36_3_B = "there has been a final determination of an acquittal"
+S36_3_B_CONV = "inadmissibility under subsections (1) to (2.1) may not be based on a conviction"
+# s. 36(3)(b) governs s. 36(2.1), but its acquittal words attach to 'a
+# conviction ... in respect of which'; s. 36(2.1) needs no conviction. The
+# law is unsettled here (principle 5): both readings, no pick.
+ACQUITTAL_NOTE = (
+    "Acquittal: s. 36(3)(b) governs s. 36(2.1) too ('" + S36_3_B_CONV + "' ... "
+    "'in respect of which " + S36_3_B + "'), but its words bar a finding based on "
+    "a CONVICTION that ended in an acquittal, and s. 36(2.1) rests on "
+    "'committing', not on a conviction. Whether a final acquittal of the entry "
+    "offence ends a s. 36(2.1) allegation is NOT SETTLED here: (A) on its words "
+    "s. 36(3)(b) does not reach a ground that needs no conviction, and the facts "
+    "are still proved on reasonable grounds to believe (s. 33); (B) the acquittal "
+    "is read as ending the allegation. Confirm before relying on either reading.")
+R228_OPENING = ("if a report in respect of a foreign national does not include any "
+                "grounds of inadmissibility other than those set out in the following "
+                "circumstances, the report shall not be referred to the Immigration "
+                "Division")
+R228_4_A = "is under 18 years of age and not accompanied by a parent or an adult legally responsible for them"
+R229_D1 = ("an offence other than one referred to in any of subparagraphs "
+           "228(1)(a.01)(i) to (v)")
+CA_8_1_A = ("designating as contraventions offences created by any enactment, other "
+            "than offences for which an offender may be prosecuted only on indictment")
+CR_S1 = ("An offence under a provision of an enactment set out in column I of the "
+         "Schedules is designated as a contravention")
+
+# ---- the guidance the memo cites (chunk ids pinned in tests/test_transborder.py) ---
+GUIDANCE_CURRENT = (
+    "ENF 5 (2025-02-20) Appendix F and ENF 6 (2025-02-20) Appendix A tabulate "
+    "A36(2.1) as 'FN only': the ID makes the deportation order under R229(1)(d.1), "
+    "the MD only for 'specific offences' under R228(1)(a.01) (enf-5-app-f-p2; "
+    "enf-6-app-a-p1, enf-6-app-a-p2). ENF 5 s. 7.4 lists 'Transborder criminality "
+    "[A36(2.1)]' among "
+    "the grounds proved on reasonable grounds to believe (enf-5-7-4; IRPA s. 33), "
+    "and ENF 5 s. 4 names the Minister of Public Safety as policy lead for A36(2.1) "
+    "(enf-5-4-p1).")
+GUIDANCE_STALE = (
+    "Older guidance still describes the repealed paragraph: ENF 1 s. 3 (2013; "
+    "enf-1-3-p30) gives 'R19: Transborder Crime' for 'paragraph A36(2)(d)' with the "
+    "old list of five Acts (no Cannabis Act), and ENF 3 s. 10.8 (2022; "
+    "enf-3-10-8-p2) and the withdrawn ENF 2 (ATIP copy, enf-2-3-p12) say the "
+    "allegation 'may only be used' in a 'present tense' scenario -- the person "
+    "cannot later be found inadmissible for a past 'on entering Canada' offence "
+    "once they withdrew or left. That limit was written for former s. 36(2)(d); "
+    "whether it still states CBSA policy under s. 36(2.1) is NOT SETTLED here "
+    "(historical guidance -- confirm before relying on it).")
+
+TABLE_HELD = (
+    "The IRPR s. 19(1)(c)-(g) table (the Firearms Act, Customs Act, Controlled "
+    "Drugs and Substances Act and Cannabis Act offences, by section and goods) is "
+    "HELD FOR REVIEW and is not in this deployment: CanLex does not say here "
+    "whether this offence is prescribed. Read IRPR s. 19(1)(c)-(g) with "
+    "canlex_get_section (act 'IRPR', section '19').")
+
+
+# ---- helpers ------------------------------------------------------------------------
+
+def _norm(text):
+    t = (text or "").lower().replace("’", "'").replace("é", "e").replace("à", "a")
+    t = re.sub(r"[^a-z0-9.\- ]", " ", t)
+    t = re.sub(r"\bthe\b", " ", t)
+    return " ".join(t.split())
+
+
+# The spellings an Act's name carries on a record or in a client's words (the code
+# audit of 2026-09-21, HELD): a parenthesised suffix ('(Canada)', '(CDSA)'), a
+# chapter citation ('R.S.C. 1985, c. C-46', 'S.C. 2018, c. 16') and a trailing
+# section ('s. 95', 'section 159(1)'). Before this any spelling the alias table did
+# not hold exactly -- 'Criminal Code of Canada', 'Customs Act (Canada)' -- resolved
+# to 'other' and the bottom line was a settled NOT A PRESCRIBED OFFENCE.
+_ACT_CITE = re.compile(r"\b(?:r\.?s\.?c\.?|s\.?c\.?|l\.?r\.?c\.?|l\.?c\.?)\s*\(?\d{4}\)?"
+                       r"(?:,?\s*(?:c|ch|chapter)\.?\s*[a-z0-9\-]+(?:\s*\(\s*\d+(?:st|nd|rd|th)"
+                       r"\s+supp\.?\s*\))?)?", re.I)
+# (the audit follow-up of 2026-09-21: 'subsection'/'paragraph' is no lift trigger of
+# its own -- '..., section 95, subsection 1' read section 1 -- it rides on the section
+# before it as 95(1))
+_ACT_TRAILING_SECTION = re.compile(
+    r"[\s,]+(?:s(?:ection|ect\.|ec\.|s\.|\.)?|art(?:icle|\.)?)\s*"
+    r"(\d+(?:\.\d+)?(?:\s*\(\s*[0-9a-z.]+\s*\))*)"
+    # (a value is a digit run or one or two letters, never a word: the value class
+    # must not overlap the keywords, or a long tail of ', subsection 1' backtracks
+    # exponentially -- the review of the follow-up; at most four of them)
+    r"((?:[\s,]+(?:sub(?:section|s\.|-s\.)|par(?:agraph|a\.?|\.)|cl(?:ause|\.))\s*"
+    r"\(?\s*(?:\d[0-9.]*|[a-z]{1,2})\s*\)?){0,4})\s*$", re.I)
+_ACT_SUBSECTION = re.compile(r"(?:sub(?:section|s\.|-s\.)|par(?:agraph|a\.?|\.)|cl(?:ause|\.))\s*"
+                             r"\(?\s*(\d[0-9.]*|[a-z]{1,2})\s*\)?", re.I)
+# the short codes match as whole tokens only; the names as word-bounded phrases
+_ACT_CODES = {"cc", "ccc", "c-46", "irpa", "i-2.5", "lipr", "f-11.6", "c-52.6", "cdsa",
+              "c-38.8", "lrcdas", "c-24.5"}
+
+
+def act_and_section(act):
+    """(act code, recognised, trailing section): resolve_act's answer and the
+    section a citation form carries after the Act ('Criminal Code s. 95' ->
+    ('C-46', True, '95')), or None when the words carry none."""
+    if not act or not act.strip():
+        return None, False, None
+    text = str(act)
+    section = None
+    m = _ACT_TRAILING_SECTION.search(text)
+    if m:
+        section = re.sub(r"\s+", "", m.group(1)) + "".join(
+            f"({s})" for s in _ACT_SUBSECTION.findall(m.group(2) or ""))
+        text = text[:m.start()]
+    text = re.sub(r"\([^)]*\)", " ", text)         # '(Canada)', '(CDSA)', '(2nd Supp.)'
+    text = _ACT_CITE.sub(" ", text).replace("&", " and ")
+    n = _norm(text)
+    if not n:
+        return None, False, section
+    tokens = set(n.split())
+    for code, meta in R19_ACTS.items():
+        names = [_norm(meta["name"])] + [a for a in meta["aliases"] if a not in _ACT_CODES]
+        if any(re.search(r"(?<![a-z0-9])" + re.escape(name) + r"(?![a-z0-9])", n)
+               for name in names if name):
+            return code, True, section
+    for code, meta in R19_ACTS.items():
+        codes = {code.lower()} | {a for a in meta["aliases"] if a in _ACT_CODES}
+        if tokens & codes:
+            return code, True, section
+    return "other", True, section
+
+
+def resolve_act(act):
+    """(act code, recognised) for an Act as the officer names it: a code of
+    R19_ACTS when it is one of the six Acts s. 19(1) names -- as a whole, or
+    inside a longer spelling ('Criminal Code of Canada', 'Customs Act
+    (Canada)', 'the Controlled Drugs & Substances Act (CDSA)', a chapter cite,
+    a trailing 's. N'); (None, False) when nothing was given; ('other', True)
+    for any other Act (the Excise Act, 2001, the Customs Tariff, a provincial
+    Act: genuinely not prescribed)."""
+    code, recognised, _ = act_and_section(act)
+    return code, recognised
+
+
+_SEC = re.compile(r"^\s*(?:(?:sub)?s(?:ection|ect\.|ec\.|s\.|\.)?\s*)?"
+                  r"(\d+(?:\.\d+)?)\s*((?:\(\s*[0-9a-z.]+\s*\)\s*)*)$", re.I)
+
+
+def parse_section(text):
+    """('320.15', ['1']) for '320.15(1)', ('124', ['1', 'b']) for
+    's. 124(1)(b)'; None when the text is not a section reference."""
+    if not text:
+        return None
+    m = _SEC.match(str(text).strip())
+    if not m:
+        return None
+    path = [p.strip().lower() for p in re.findall(r"\(\s*([0-9a-z.]+)\s*\)", m.group(2) or "")]
+    return m.group(1), path
+
+
+def section_relation(given, key):
+    """'match' when the officer's section is the rule's provision or inside
+    it; 'partial' when the officer gave less than the rule needs ('320.15'
+    against '320.15(1)'); 'no' otherwise."""
+    kb = parse_section(key)
+    if not given or not kb:
+        return "no"
+    gb, gp = given
+    if gb != kb[0]:
+        return "no"
+    kp = kb[1]
+    n = min(len(gp), len(kp))
+    if gp[:n] != kp[:n]:
+        return "no"
+    return "match" if len(gp) >= len(kp) else "partial"
+
+
+def section_label(sec):
+    if not sec:
+        return None
+    return sec[0] + "".join(f"({p})" for p in sec[1])
+
+
+def load_table(path=None):
+    """The curated s. 19(1)(c)-(g) table, or None when it is absent (a
+    composed release ships without held files)."""
+    from canlex.config import DATA_DIR
+    path = path or (DATA_DIR / "curated" / TABLE_FILE)
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+
+def mode_from_pairing(p):
+    """The Canadian offence's mode as the curated pairing records it
+    (summary_only / hybrid / a maximum), or None."""
+    if not p or p.get("equivalent") is False:
+        return None
+    if p.get("summary_only") is True:
+        return "summary-only"
+    if p.get("hybrid") is True:
+        return "hybrid"
+    if p.get("hybrid") is False and type(p.get("max_years")) is int:
+        return "indictable"
+    return None
+
+
+def pairing_act(p):
+    """The one Act a pairing's verify rows cite, or None when they cite
+    several or none."""
+    acts = {v.get("act") for v in (p or {}).get("verify") or []}
+    return acts.pop() if len(acts) == 1 else None
+
+
+# ---- contraventions (IRPA s. 36(3)(e)(i)) -------------------------------------------
+
+def _schedule_act(chunk):
+    """The Act a Contraventions Regulations schedule is for: its title, or
+    (for a schedule the ingester titled only 'Schedule X') the first line
+    naming it."""
+    title = (chunk.get("marginal_note") or "").strip()
+    if title and not title.lower().startswith("schedule"):
+        return title
+    for line in (chunk.get("text") or "").splitlines():
+        line = line.strip()
+        if "|" in line:          # a table row is never an Act's name
+            return ""
+        if line and not line.startswith("(Section") and line.lower() != "part i":
+            return line
+    return ""
+
+
+def contravention_schedules(processed_dir=None):
+    """(the Acts the SOR-96-313 schedules are for, the corpus current_to date),
+    or (None, None) when the Regulations are not in the corpus."""
+    from canlex.config import DATA_DIR
+    path = (processed_dir or (DATA_DIR / "processed")) / "SOR-96-313.json"
+    try:
+        chunks = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None, None
+    acts, current, seen = [], None, set()
+    for c in chunks:
+        if c.get("lang", "en") != "en":
+            continue
+        current = current or c.get("current_to")
+        sec = str(c.get("section", ""))
+        # One Act per schedule, read from the schedule's FIRST chunk: a
+        # continuation chunk titled only 'Schedule X' starts with a table row.
+        if not sec.lower().startswith("schedule") or sec in seen:
+            continue
+        seen.add(sec)
+        a = _schedule_act(c)
+        if a and a not in acts and a.lower() != "[repealed]":
+            acts.append(a)
+    return acts, current
+
+
+def contravention_check(act_code, schedules=None):
+    """{'status', 'text'} for the s. 36(3)(e)(i) carve-out: 'not-designated'
+    when no schedule of the Contraventions Regulations is for this Act,
+    'check-schedule' when one is, 'not-checked' when the Regulations are not
+    in the corpus or the Act is unknown."""
+    acts, current = schedules if schedules is not None else contravention_schedules()
+    head = (f"Contraventions carve-out (IRPA s. 36(3)(e)(i)): '{S36_3_E}' "
+            f"'{S36_3_E_I}'.")
+    if acts is None:
+        return {"status": "not-checked",
+                "text": head + " The Contraventions Regulations are not in this "
+                        "deployment's corpus, so this was not checked -- read "
+                        "SOR-96-313 before relying on the allegation."}
+    if act_code not in R19_ACTS:
+        return {"status": "not-checked",
+                "text": head + " Not checked: the Act is not one IRPR s. 19(1) names."}
+    name = R19_ACTS[act_code]["name"]
+    if any(_norm(a) == _norm(name) for a in acts):
+        return {"status": "check-schedule",
+                "text": head + f" The Contraventions Regulations (SOR-96-313, current "
+                        f"to {current}) have a schedule for the {name}: read it for "
+                        "this provision -- an offence designated there cannot ground "
+                        "s. 36(2.1)."}
+    text = (head + f" The Contraventions Regulations (SOR-96-313, current to {current}) "
+            f"designate offences by schedule ('{CR_S1}'), and none of their "
+            f"{len(acts)} schedules is for the {name}: no offence under it is "
+            "designated, so the carve-out does not apply.")
+    if act_code == "C-46":
+        text += (" (Contraventions Act s. 8(1)(a) also excludes 'offences for which an "
+                 "offender may be prosecuted only on indictment'.)")
+    if act_code == "C-24.5":
+        text += (" A Cannabis Act ticket (ss. 51-52) is that Act's own procedure, not a "
+                 "designation under the Contraventions Act, so s. 36(3)(e)(i) does not "
+                 "reach it on its words; how a ticketable amount is treated at the "
+                 "border is NOT SETTLED here.")
+    return {"status": "not-designated", "text": text}
+
+
+# ---- the assessment -------------------------------------------------------------------
+
+def _open_class(code, mode):
+    """(prescribed, how) for s. 19(1)(a)/(b): indictable and hybrid offences
+    are prescribed (hybrids deemed indictable by s. 19(2)); a summary-only
+    offence is not."""
+    para = R19_ACTS[code]["paragraph"]
+    name = R19_ACTS[code]["name"]
+    if mode == "indictable":
+        return True, f"IRPR s. {para}: 'any indictable offence under the {name}'"
+    if mode == "hybrid":
+        return True, (f"IRPR s. {para} with s. 19(2): '{R19_2}' -- a hybrid offence is "
+                      "prescribed")
+    if mode == "summary-only":
+        return False, (f"IRPR s. {para} prescribes only indictable offences under the "
+                       f"{name}; an offence that may only be prosecuted summarily is "
+                       "not prescribed")
+    return None, None
+
+
+def _md(code, sec, goods):
+    """(md, subparagraph, readings): whether IRPR s. 228(1)(a.01) lets the
+    Minister's delegate make the deportation order. md is None when the
+    officer's section or goods leave it open."""
+    rules = [r for r in MD_ORDERABLE if r[1] == code]
+    if not rules:
+        return False, None, []
+    if sec is None:
+        return None, None, [f"s. {k}" + (f" with goods: {r[3]}" if r[3] else "")
+                            + f" (IRPR s. {r[0]})" for r in rules for k in r[2]]
+    partial = []
+    for sub, _, keys, cond in rules:
+        for k in keys:
+            rel = section_relation(sec, k)
+            if rel == "no":
+                continue
+            gm = ("match" if cond is None or goods == cond
+                  else "partial" if goods is None else "no")
+            if rel == "match" and gm == "match":
+                return True, sub, []
+            if gm != "no":
+                partial.append(f"s. {k}" + (f" with goods: {cond}" if cond else "")
+                               + f" (IRPR s. {sub})")
+    return (None if partial else False), None, partial
+
+
+def _table_rows(table, code):
+    return [r for r in (table or {}).get("rows", []) if r.get("act") == code]
+
+
+def _from_table(table, code, sec, goods):
+    """(prescribed, row, readings, needs) from the curated (c)-(g) table."""
+    rows = _table_rows(table, code)
+    if sec is None:
+        readings = [f"s. {r['section']}" + (f" / goods {r['goods']}" if r.get("goods") else "")
+                    + f": {'prescribed' if r.get('prescribed', True) else 'NOT prescribed'}"
+                    + (f" (IRPR s. {r['paragraph']})" if r.get("paragraph") else "")
+                    for r in rows]
+        return None, None, readings, [{"fact": "section", "parameter": "section",
+                                       "question": f"Which section of the "
+                                                   f"{R19_ACTS[code]['name']} was contravened?"}]
+    hits = [(section_relation(sec, r["section"]), r) for r in rows]
+    exact = [r for rel, r in hits if rel == "match"]
+    partial = [r for rel, r in hits if rel == "partial"]
+    # A penalty section (Customs Act s. 160(1), CDSA s. 46) punishes the
+    # contravention of other provisions: the provision contravened decides.
+    punished = [r for r in rows if r.get("punished_under")
+                and section_relation(sec, r["punished_under"]) != "no"]
+    if punished and not exact and not partial:
+        readings = [f"s. {r['section']}" + (f" / goods {r['goods']}" if r.get("goods") else "")
+                    + (" (prescribed" + (f", IRPR s. {r['paragraph']})" if r.get("paragraph")
+                                         else ")")
+                       if r.get("prescribed", True) else " (NOT prescribed)")
+                    for r in punished]
+        return None, None, readings, [{
+            "fact": "section", "parameter": "section",
+            "question": f"s. {section_label(sec)} of the {R19_ACTS[code]['name']} punishes "
+                        "contraventions of other provisions: give the provision "
+                        "contravened (for example '12' for failing to report goods, "
+                        "'156', '46.3(1)')."}]
+    if exact:
+        goods_rows = [r for r in exact if r.get("goods")]
+        if goods_rows:
+            chosen = [r for r in goods_rows if r["goods"] == goods]
+            if goods is None:
+                readings = [f"goods {r['goods']}: "
+                            + ("prescribed" if r.get("prescribed", True) else "NOT prescribed")
+                            + (f" (IRPR s. {r['paragraph']})" if r.get("paragraph") else "")
+                            for r in goods_rows]
+                if not any(r["goods"] == "other" for r in goods_rows):
+                    readings.append("any other goods: NOT prescribed under this provision")
+                return None, None, readings, [{
+                    "fact": "goods", "parameter": "goods",
+                    "question": "What were the goods? IRPR s. 19(1)(d)-(e) prescribe "
+                                "this offence only for the goods classes listed."}]
+            if chosen:
+                r = chosen[0]
+                return bool(r.get("prescribed", True)), r, [], []
+            return False, None, [], []
+        r = exact[0]
+        return bool(r.get("prescribed", True)), r, [], []
+    if partial:
+        readings = [f"s. {r['section']}" + (f" / goods {r['goods']}" if r.get("goods") else "")
+                    + (" (prescribed)" if r.get("prescribed", True) else " (NOT prescribed)")
+                    for r in partial]
+        return None, None, readings, [{"fact": "section", "parameter": "section",
+                                       "question": "Give the subsection or the provision "
+                                                   "contravened: " + "; ".join(readings)}]
+    return False, None, [], []
+
+
+def assess(act=None, section=None, mode=None, goods=None, status=None, offence=None,
+           pairings=None, table=None, match=None):
+    """The transborder result as a dict (see server.canlex_transborder).
+    pairings: the curated equivalency pairings (for the mode of a Criminal
+    Code / IRPA offence named in words); table: the curated (c)-(g) table or
+    None; match: screening.match_pairings (injected, so this module has no
+    matcher dependency)."""
+    person = status or "foreign-national"
+    code, recognised, lifted = act_and_section(act)
+    lifted_note = None
+    if lifted and not (section and str(section).strip()):
+        # 'Criminal Code s. 95' with no section given: the section rides on the Act
+        section, lifted_note = lifted, (f"Section '{lifted}' read from the act words "
+                                        f"('{act}'); give it as `section` to be sure.")
+    sec = parse_section(section)
+    out = {"person": person, "act": code if code != "other" else None,
+           "act_given": act, "act_name": R19_ACTS.get(code, {}).get("name"),
+           "section": section_label(sec), "section_given": section,
+           "mode": mode, "mode_source": "given" if mode else None,
+           "goods": goods, "paragraph": None, "prescribed": None,
+           "prescribed_how": None, "readings": [], "needs": [], "pairing": None,
+           "md_orderable": None, "md_provision": None, "md_readings": [],
+           "removal": None, "removal_provision": None, "table": "present" if table else "absent",
+           "notes": []}
+    if lifted_note:
+        out["notes"].append(lifted_note)
+    if section and sec is None:
+        out["notes"].append(f"'{section}' is not read as a section number (write it as "
+                            "'95', '320.15(1)' or '124(1)(b)').")
+    # A Criminal Code / IRPA offence named only in words: the curated pairing
+    # (and its verified mode flags) may name the Act and the mode.
+    if offence and pairings and match and (code in (None, "C-46", "I-2.5")):
+        scored = match(offence, pairings)
+        if scored:
+            p = scored[0][1]
+            pact = pairing_act(p)
+            if pact in ("C-46", "I-2.5") and (code is None or code == pact):
+                out["pairing"] = p["id"]
+                if code is None:
+                    code, out["act"] = pact, pact
+                    out["act_name"] = R19_ACTS[pact]["name"]
+                    out["notes"].append(f"Act read from the curated pairing '{p['id']}' "
+                                        f"({p['canadian_offence']}) -- confirm it.")
+                if mode is None and mode_from_pairing(p):
+                    mode = out["mode"] = mode_from_pairing(p)
+                    out["mode_source"] = f"pairing:{p['id']}"
+    if code is None:
+        out["needs"].append({"fact": "act", "parameter": "act",
+                             "question": "Under which Act is the offence (Criminal "
+                                         "Code, IRPA, Firearms Act, Customs Act, CDSA, "
+                                         "Cannabis Act, or another)?"})
+    elif code == "other":
+        out["prescribed"] = False
+        out["prescribed_how"] = (
+            f"IRPR s. 19(1) prescribes offences under six Acts only -- the "
+            f"Criminal Code, IRPA, the Firearms Act, the Customs Act, the CDSA and "
+            f"the Cannabis Act -- and '{act}' is not read as one of them. If it is "
+            "one of the six, re-run with its name or code.")
+    elif R19_ACTS[code]["open"]:
+        out["paragraph"] = R19_ACTS[code]["paragraph"]
+        pres, how = _open_class(code, mode)
+        out["prescribed"], out["prescribed_how"] = pres, how
+        if pres is None:
+            name = R19_ACTS[code]["name"]
+            out["readings"] = [f"indictable or hybrid (s. 19(2) deems a hybrid "
+                               f"indictable): prescribed under IRPR s. {out['paragraph']}",
+                               "summary-only (may ONLY be prosecuted summarily): NOT "
+                               "prescribed"]
+            out["needs"].append({"fact": "mode", "parameter": "mode",
+                                 "question": f"How may this {name} offence be "
+                                             "prosecuted: indictable, hybrid or "
+                                             "summary-only? (read the section's "
+                                             "punishment clause)"})
+    elif table is None:
+        out["paragraph"] = R19_ACTS[code]["paragraph"]
+        out["notes"].append(TABLE_HELD)
+    else:
+        pres, row, readings, needs = _from_table(table, code, sec, goods)
+        out["prescribed"], out["readings"] = pres, readings
+        out["needs"] += needs
+        if row:
+            out["paragraph"] = row.get("paragraph")
+            out["prescribed_how"] = row.get("note") or (
+                f"IRPR s. {row['paragraph']}" if row.get("paragraph") else None)
+            out["row"] = row.get("id")
+        elif pres is False:
+            out["prescribed_how"] = (f"not among the offences IRPR s. "
+                                     f"{R19_ACTS[code]['paragraph']} prescribes under the "
+                                     f"{R19_ACTS[code]['name']} (as CanLex tabulates "
+                                     "them, held for review)")
+    # The delegate's jurisdiction (s. 228(1)(a.01)) -- a rule on the section,
+    # whether or not the (c)-(g) table is present.
+    if code in R19_ACTS and out["prescribed"] is not False:
+        md, sub, mreadings = _md(code, sec, goods)
+        out["md_orderable"], out["md_readings"] = md, mreadings
+        if sub:
+            out["md_provision"] = f"IRPR s. {sub}"
+        if md is None:
+            fact = "goods" if (sec is not None and code == "C-52.6" and goods is None) else "section"
+            if not any(n["fact"] == fact for n in out["needs"]):
+                out["needs"].append({
+                    "fact": fact, "parameter": fact,
+                    "question": ("What were the goods? The delegate's order turns on them: "
+                                 if fact == "goods" else
+                                 "Give the section and subsection: the Minister's delegate "
+                                 "makes the order only for ")
+                                + "; ".join(mreadings)})
+    if (code == "C-52.6" and sec and sec[0] == "160"
+            and not any(n["fact"] == "section" for n in out["needs"])):
+        out["needs"].append({"fact": "section", "parameter": "section",
+                             "question": "Customs Act s. 160(1) punishes many "
+                                         "contraventions: give the provision "
+                                         "contravened ('12' for failing to report "
+                                         "goods, '156', or '159(1)' for smuggling)."})
+    if person == "permanent-resident":
+        out["branch"] = "not-reached-pr"
+    elif out["prescribed"] is True:
+        out["branch"] = "transborder-criminality"
+    elif out["prescribed"] is False:
+        out["branch"] = "not-prescribed"
+    elif code in R19_ACTS and not R19_ACTS[code]["open"] and table is None:
+        out["branch"] = "table-held"
+    else:
+        out["branch"] = "not-settled"
+    # The removal path is reported only once the offence is prescribed; while
+    # that is open the memo gives it conditionally.
+    if person == "foreign-national" and out["prescribed"] is True:
+        if out["md_orderable"] is True:
+            out["removal"], out["removal_provision"] = "md-deportation-order", out["md_provision"]
+        elif out["md_orderable"] is False:
+            out["removal"], out["removal_provision"] = "id-referral", "IRPR s. 229(1)(d.1)"
+    return out
+
+
+# ---- the memo ---------------------------------------------------------------------
+
+def _yn(v):
+    return {True: "YES", False: "NO", None: "NOT SETTLED"}[v]
+
+
+def render(r, offence=None, trp_fee=None, family_line=None, contravention=None):
+    """The memo lines (no banner, no CANLEX_RESULT line) for an assess()
+    result. trp_fee: the IRPR s. 298 amount read from the corpus;
+    family_line: the s. 42 family-member line; contravention: a
+    contravention_check() result."""
+    lines = []
+    person = r["person"]
+    act_txt = (f"{r['act_name']} ({r['act']})" if r.get("act_name")
+               else (r.get("act_given") or "not given"))
+    facts = [f"Offence: {offence or 'not described'}",
+             f"Act: {act_txt}; section: {r['section'] or r.get('section_given') or 'not given'}",
+             f"Mode: {r['mode'] or 'not given'}"
+             + (f" (from the curated pairing '{r['mode_source'][8:]}' -- confirm it)"
+                if (r.get("mode_source") or "").startswith("pairing:") else ""),
+             f"Goods: {r['goods'] or 'not given'}",
+             f"Person: {person}"]
+    lines += ["\n".join(facts), ""]
+    lines += ["## Who s. 36(2.1) reaches",
+              f"IRPA s. 36(2.1): '{S36_2_1}.' It reaches a foreign national only, "
+              "for an offence committed ON ENTERING Canada; it needs no conviction "
+              "('committing'), and the facts are proved on reasonable grounds to "
+              "believe (IRPA s. 33). Former s. 36(2)(d) is repealed (2023, c. 32, "
+              "s. 53).",
+              ACQUITTAL_NOTE]
+    if person == "permanent-resident":
+        lines.append("PERMANENT RESIDENT: s. 36(2.1) does not reach a permanent "
+                     "resident. A permanent resident is reached for this conduct only "
+                     "under s. 36(1)(a) once convicted in Canada (a maximum of at least "
+                     "10 years, or more than six months imposed) -- s. 36(2)(a) reaches "
+                     "foreign nationals only. s. 37 (organized criminality) is a "
+                     "separate ground.")
+    lines += ["", "## Is it a prescribed offence? (IRPR s. 19)",
+              f"Prescribed: {_yn(r['prescribed'])}"
+              + (f" -- {r['prescribed_how']}" if r.get("prescribed_how") else "")]
+    if r["readings"]:
+        lines.append("Each reading:")
+        lines += [f"- {x}" for x in r["readings"]]
+    for n in r["notes"]:
+        lines.append(f"NOTE: {n}")
+    if r.get("row"):
+        lines.append(f"(Row '{r['row']}' of the IRPR s. 19(1)(c)-(g) table -- HELD FOR "
+                     "REVIEW.)")
+    lines += ["", "## Removal path"]
+    if person == "permanent-resident":
+        lines.append("Not reached: no s. 36(2.1) report lies against a permanent resident.")
+    elif r["prescribed"] is False:
+        lines.append("None on this ground: the offence is not prescribed.")
+    else:
+        cond = "" if r["prescribed"] else "IF the offence is prescribed (not settled above): "
+        if r["md_orderable"] is True:
+            lines.append(
+                f"{cond}The Minister's delegate makes a DEPORTATION ORDER ({r['md_provision']}): "
+                "the offence is one R228(1)(a.01) lists, so the report is not referred "
+                "to the Immigration Division -- but only where the report includes no "
+                f"other ground (R228(1): '{R228_OPENING}'), and not for a person who "
+                f"'{R228_4_A}' or who cannot appreciate the proceedings and is "
+                "unaccompanied (R228(4)).")
+        elif r["md_orderable"] is False:
+            lines.append(
+                f"{cond}Referred to the Immigration Division (IRPA s. 44(2)), which makes a "
+                "DEPORTATION ORDER (IRPR s. 229(1)(d.1): transborder criminality for "
+                f"'{R229_D1}'). The delegate cannot make the order: the offence is not "
+                "one R228(1)(a.01) lists.")
+        else:
+            lines.append(f"{cond}NOT SETTLED: the delegate makes the deportation order (IRPR "
+                         "s. 228(1)(a.01)) only for the offences it lists; every other "
+                         "prescribed offence is referred to the Immigration Division "
+                         "(IRPR s. 229(1)(d.1)). The delegate's list for this Act:")
+            lines += [f"- {x}" for x in r["md_readings"]] or ["- (none)"]
+        lines.append("At a port of entry the alternatives are allowing the person to "
+                     "withdraw the application to enter and leave (IRPR s. 42 -- once a "
+                     "s. 44(1) report is being prepared, only if the Minister decides not "
+                     "to make a removal order or not to refer the report, s. 42(2)) or a "
+                     "temporary resident permit (below).")
+    lines += ["", "## Rehabilitation",
+              "None: s. 36(3)(c) cures only 'the matters referred to in paragraphs "
+              "(1)(b) and (c) and (2)(b) and (c)' -- s. 36(2.1) is not among them, so "
+              "neither deemed nor individual rehabilitation (IRPR ss. 17-18) reaches "
+              "transborder criminality."]
+    if contravention:
+        lines += ["", "## Contraventions", contravention["text"]]
+    lines += ["", "## Next steps"]
+    if person == "foreign-national" and r["prescribed"] is not False:
+        lead = "" if r["prescribed"] else "If the offence is prescribed (above): "
+        name = (offence or "the offence").strip()
+        cite = (f" contrary to section {r['section']} of the {r['act_name']}"
+                if r.get("section") and r.get("act_name") else
+                f" under the {r['act_name']}" if r.get("act_name") else "")
+        para = (f", an offence prescribed by paragraph {r['paragraph']} of the Immigration "
+                "and Refugee Protection Regulations" if r.get("paragraph")
+                and r["prescribed"] else "")
+        deemed = (" (a hybrid offence, deemed indictable by IRPR s. 19(2))"
+                  if r.get("mode") == "hybrid" and r.get("paragraph") in ("19(1)(a)", "19(1)(b)")
+                  else "")
+        lines.append(f"{lead}Draft allegation paragraph (s. 36(2.1) wording; ENF 5 s. 13 "
+                     "sets out the report narrative): the person is a "
+                     "foreign national inadmissible under IRPA subsection 36(2.1) for "
+                     f"{ALLEGATION_2_1}, namely {name}{cite}{para}{deemed}, committed on "
+                     "entering Canada on <date> at <port of entry>. Record in the "
+                     "narrative the facts of entry, the evidence (statutory declaration, "
+                     "notebook, seizure record) and the section contravened.")
+        lines.append("Temporary resident permit (IRPA s. 24): available at the officer's "
+                     "discretion where entry is justified in the circumstances, subject "
+                     "to the s. 24(3.1)-(5) bars; the processing fee under IRPR s. 298 is "
+                     + (f"${trp_fee}" if trp_fee else "set out in IRPR s. 298") + ".")
+    elif person == "foreign-national":
+        lines.append("No s. 36(2.1) allegation: the offence is not prescribed. Consider "
+                     "whether another ground applies (a conviction under s. 36(2)(a), "
+                     "s. 41 non-compliance).")
+    if family_line and person == "foreign-national":
+        lines.append(family_line)
+    lines += ["", "## Guidance", GUIDANCE_CURRENT, GUIDANCE_STALE]
+    return lines
+
+
+def bottom_line(r):
+    person = r["person"]
+    if person == "permanent-resident":
+        return ("NOT REACHED -- s. 36(2.1) does not reach a permanent resident; "
+                "s. 36(1)(a) only once convicted.")
+    if r["branch"] == "transborder-criminality":
+        how = ("the delegate makes the deportation order (" + r["md_provision"] + ")"
+               if r["md_orderable"] is True else
+               "referral to the Immigration Division (IRPR s. 229(1)(d.1))"
+               if r["md_orderable"] is False else
+               "the delegate or the Immigration Division, by section (not settled)")
+        return (f"TRANSBORDER CRIMINALITY (IRPA s. 36(2.1); IRPR s. {r['paragraph']}) "
+                f"if committed on entering Canada -- {how}; no rehabilitation; TRP "
+                "available.")
+    if r["branch"] == "not-prescribed":
+        return "NOT A PRESCRIBED OFFENCE -- no s. 36(2.1) ground on these facts."
+    if r["branch"] == "table-held":
+        return ("NOT ASSESSED -- the IRPR s. 19(1)(c)-(g) table is held for review; read "
+                "s. 19(1) for this offence.")
+    return "NOT SETTLED -- " + "; ".join(n["question"] for n in r["needs"]) if r["needs"] \
+        else "NOT SETTLED."