Deploy: audit round 1 -- whole-section get_section, Latin-1 request bodies, rehab s. 17 proviso, rerank pool cap, applicability caveats

.gitignore

@@ -7,3 +7,4 @@ canlii_key.txt
 data/citator_*.json
 US-Dispositions-Reference.pdf
 *.bundle
+data/logs/

RUNBOOK.md

@@ -114,9 +114,13 @@ JSON that would read as an empty corpus and disable the guard next time.
 
 ## Deploys
 
-- **MCP Space** (`Beemer0/CanLex`): commit, then `git push space main`.
-  The Space rebuilds automatically. `embeddings.npz` ships in-repo via
-  git-LFS — a fresh clone needs `git lfs install`.
+- **MCP Space** (`Beemer0/CanLex`): commit on `main`, then ship it with the
+  squashed `deploy`-branch recipe in "Deploying to the Hugging Face Space"
+  above -- never `git push space main`: the Space's history diverged at the
+  2026-08-31 squash, so that push is rejected, and forcing it would re-upload
+  every historical `embeddings.npz`. The Space rebuilds automatically after
+  the push. `embeddings.npz` ships in-repo via git-LFS — a fresh clone needs
+  `git lfs install`.
 - **Webapp Space** (`Beemer0/Canlex-web`): separate repo. Clone it, copy
   `webapp/app.py` + `webapp/requirements.txt` + `webapp/Dockerfile` +
   `webapp/README.md` over, commit, push. Do NOT use `hf upload` — HF gates
@@ -283,3 +287,20 @@ argued about.
   of "what is a pre-removal risk assessment" entirely. If a new corpus
   family lands, check which field carries its document identity before
   trusting the caps.
+- **The rerank pool is source-capped (`POOL_CAP`, 2026-09-06).** The
+  cross-encoder only ever sees `RERANK_POOL` (16) fused candidates, and
+  `RECALL_CAP` (20) let one deep decision take 14 of those slots: on "can an
+  officer search a traveller's cell phone at the port of entry without a
+  warrant" the pool was 14 Fearon paragraphs plus two unrelated "without
+  warrant" marginal notes, while Canfield, Pike and ENF 12 s. 6.15 sat at
+  fused ranks 19-32 and were never scored. `_fill_pool` now admits at most
+  `POOL_CAP` (5) chunks per capped source while filling the pool; `_diversify`
+  still governs display afterwards. Eval-gated: no slice moved. The
+  device-vocabulary bridge in synonyms.py is the other half of that fix.
+- **Some law in the corpus does not bind its audience (`_INAPPLICABLE`).**
+  Canada Labour Code Part III excludes federal departments (s. 167(1)(d)),
+  and FPSLRA Part 2.1 is RCMP-only, yet both out-BM25'd the FB agreement on
+  steward questions (CLC s. 173.1 over FB art. 25.21 for shift-change
+  notice). Those chunks take `INAPPLICABLE_PENALTY` in the fusion and carry
+  an APPLICABILITY caveat whenever they are rendered. Extend the table when a
+  new instrument with a public-service carve-out lands.

canlex/index.py

@@ -25,6 +25,24 @@ RERANK_POOL = int(os.environ.get("CANLEX_RERANK_POOL", "16"))
                     # low-fusion candidates the promote-only rerank can lift).
                     # Swept 2026-06: 50 -> 16 (with doc cap 3000 -> 1000) held
                     # the 159-Q eval exactly (MRR +0.01) at ~5.5x lower latency.
+POOL_CAP = int(os.environ.get("CANLEX_POOL_CAP", "8"))
+                    # max chunks one capped source (a decision, a memorandum)
+                    # may hold in the rerank pool. The pool is the cross-
+                    # encoder's whole world: with RECALL_CAP=20 a deep
+                    # decision can take 14 of 16 slots (Fearon, 88 paragraphs
+                    # on cell-phone searches, for a device-search query), so
+                    # the governing border authorities sitting at fused ranks
+                    # 19-32 (Canfield, Pike, ENF 12 s. 6.15) are never scored
+                    # at all -- _diversify runs after the rerank and can only
+                    # reorder what was scored. Capping while the pool fills
+                    # backfills those slots with the next sources. Swept
+                    # 2026-09-06 over {5,6,8,10,12,16,off}: 5 let two
+                    # incidental FB articles into the top 5 of the
+                    # discipline question, so _ensure_primary no longer
+                    # pulled art. 17 in (absent from the top 20); 6-16 keep
+                    # it at 3 with Canfield at 4 on the device query. 8 is
+                    # the comfortable middle. Env-sweepable; eval-gate any
+                    # change.
 MN_WEIGHT = float(os.environ.get("CANLEX_MN_WEIGHT", "0.0024"))
                     # title-match boost per unit of idf-weighted overlap between
                     # the query and a candidate's marginal note (section title)
@@ -50,6 +68,14 @@ REPEALED_PENALTY = float(os.environ.get("CANLEX_REPEALED_PENALTY", "0.008"))
                     # inform): a repealed section's on-topic title must not
                     # beat live law in a close contest -- the CDSA-to-Cannabis
                     # -Act migration is the live risk zone
+INAPPLICABLE_PENALTY = float(os.environ.get("CANLEX_INAPPLICABLE_PENALTY", "0.004"))
+                    # fusion penalty on provisions the corpus's own audience
+                    # is excluded from (see _INAPPLICABLE): Canada Labour
+                    # Code Part III does not apply to federal departments
+                    # (s. 167(1)(d)), yet its hours-of-work sections out-
+                    # score the FB agreement article that actually governs
+                    # a shift-change grievance. The chunks stay retrievable
+                    # and carry an applicability caveat when shown.
 BACKMATTER_PENALTY = float(os.environ.get("CANLEX_BACKMATTER_PENALTY", "0.004"))
                     # likewise for a collective agreement's back-matter
                     # (memoranda, letters of understanding) vs its numbered articles
@@ -112,6 +138,45 @@ _APPENDIX_REF = re.compile(r"\bAppendi(?:x|ces)\s+([A-Za-z])\b(?!\s+of\b)",
 # A directive chunk that *is* an appendix: its title opens "Appendix C ...".
 _APPENDIX_HEAD = re.compile(r"Appendix\s+([A-Za-z])\b", re.IGNORECASE)
 
+# Provisions that are law, but not law binding the corpus's audience --
+# federal public-service (FB) employees. Keyed (act_code, Part prefixes);
+# the value is the caveat rendered with every such chunk. These stay
+# retrievable (the question may be about them) but take INAPPLICABLE_PENALTY
+# in the fusion and can never be handed to a steward as the governing text.
+_INAPPLICABLE = {
+    ("L-2", ("PART III", "PARTIE III")): (
+        "Part III of the Canada Labour Code (standard hours, wages, "
+        "vacations, holidays, leave, unjust dismissal) does NOT apply to "
+        "federal departments and agencies such as the CBSA -- s. 167(1)(d) "
+        "excludes 'a department as defined in the Financial Administration "
+        "Act'. For FB employees the governing texts are the FB collective "
+        "agreement (hours of work art. 25, leave arts. 34-52, grievances "
+        "art. 18) and the FPSLRA; only Part II (occupational health and "
+        "safety) of the Code reaches the public service."),
+    ("P-33.3", ("PART 2.1", "PARTIE 2.1")): (
+        "Part 2.1 of the FPSLRA applies only to RCMP members and reservists "
+        "(ss. 238.01-238.29). It does not govern FB employees, whose "
+        "grievance rights are in Part 2 (ss. 208-238) and the FB collective "
+        "agreement art. 18."),
+}
+
+
+def _applicability(chunk):
+    """The _INAPPLICABLE caveat for a legislation chunk, or None."""
+    if chunk.get("doc_type", "legislation") != "legislation":
+        return None
+    part = (chunk.get("part") or "").upper()
+    for (code, prefixes), note in _INAPPLICABLE.items():
+        if chunk.get("act_code") == code and part.startswith(prefixes):
+            return note
+    return None
+
+
+def _piece_no(chunk):
+    """Ordinal of a split section's piece ('-p3' -> 3; unsplit -> 0)."""
+    m = _PIECE_ID.search(chunk.get("id", ""))
+    return int(m.group()[2:]) if m else 0
+
 
 _STEMMER = snowballstemmer.stemmer("english")
 _STEM_CACHE = {}
@@ -322,8 +387,13 @@ class LegislationIndex:
         self._is_backmatter = []
         self._is_nif = []
         self._is_repealed = []
+        self._inapplicable = []
         for c in self.chunks:
             self._note_tokens.append(set(tokenize(topical_title(c))))
+            note = _applicability(c)
+            self._inapplicable.append(bool(note))
+            if note:
+                c["applicability"] = note   # rendered by server._format_section
             self._is_regulation.append(
                 c.get("doc_type", "legislation") == "legislation"
                 and c["act_code"].startswith(("SOR", "C.R.C")))
@@ -649,6 +719,27 @@ class LegislationIndex:
                 break
         return out
 
+    def _fill_pool(self, candidates):
+        """The rerank pool: the first RERANK_POOL fused candidates, holding
+        at most POOL_CAP chunks of any one capped source. Skipped surplus
+        frees slots that the next-ranked chunks of OTHER sources backfill,
+        so the cross-encoder sees the field, not one deep decision."""
+        pool, counts = [], defaultdict(int)
+        for idx in candidates:
+            key = self._source_key(idx)
+            # Pieces of ONE long provision (FB art. 17 is eight) are not a
+            # monopoly to break: the reranker must see every piece to find
+            # the one that answers, and _diversify shows only two of them
+            # anyway. Capping them here lost the discipline article outright.
+            if key is not None and key[0] != "piece":
+                counts[key] += 1
+                if counts[key] > POOL_CAP:
+                    continue
+            pool.append(idx)
+            if len(pool) >= RERANK_POOL:
+                break
+        return pool
+
     def _diversify(self, ordered):
         """Reorder so no single case, memorandum, agreement or directive can
         monopolise the results: once a source has contributed SOURCE_CAP chunks,
@@ -714,6 +805,8 @@ class LegislationIndex:
                     score -= REG_PENALTY
                 if self._is_backmatter[idx]:
                     score -= BACKMATTER_PENALTY
+                if self._inapplicable[idx]:
+                    score -= INAPPLICABLE_PENALTY
                 return score
             # Sort by title-match descending, then by original fusion order as
             # a tiebreak (stable sort: keep the original rest order).
@@ -895,6 +988,8 @@ class LegislationIndex:
                 fused[idx] -= NIF_PENALTY
             elif self._is_repealed[idx]:
                 fused[idx] -= REPEALED_PENALTY
+            elif self._inapplicable[idx]:
+                fused[idx] -= INAPPLICABLE_PENALTY
             elif self._is_regulation[idx]:
                 fused[idx] -= REG_PENALTY
             elif self.chunks[idx].get("doc_type") == "memorandum":
@@ -916,7 +1011,7 @@ class LegislationIndex:
         # unreliable on long statutory text (it can score the right section
         # negative), so its power to demote a candidate is deliberately removed.
         if self.reranker:
-            pool = candidates[:RERANK_POOL]
+            pool = self._fill_pool(candidates)
             docs = [self._rerank_doc(i) for i in pool]
             ce_orig = self.reranker.score(query, docs)
             # A repaired query is scored alongside the original and each
@@ -939,7 +1034,8 @@ class LegislationIndex:
                 sorted(pool, key=ce.get, reverse=True))}
             pool.sort(key=lambda i: (min(fusion_rank[i], rerank_rank[i]),
                                      fusion_rank[i]))
-            candidates = pool + candidates[RERANK_POOL:]
+            in_pool = set(pool)
+            candidates = pool + [i for i in candidates if i not in in_pool]
 
         # Explicit section references are pinned to the very top -- using the
         # same Act-mentioned constraint as the recall step above, for the same
@@ -987,6 +1083,23 @@ class LegislationIndex:
                 fallback = fallback or c
         return fallback
 
+    def get_section_parts(self, act, section, lang="en"):
+        """Every stored piece of one section, in order -- the chunk
+        get_section returns plus its '-p2', '-p3', ... siblings (same base
+        id, same language). Long provisions are split for embedding, and
+        serving only the first piece silently dropped IRPA s. 36(3) and
+        IRPR s. 18(2): the subsections the screening tools point at.
+        Returns [] when the section is not found."""
+        first = self.get_section(act, section, lang)
+        if first is None:
+            return []
+        base = _PIECE_ID.sub("", first["id"])
+        first_lang = first.get("lang", "en")
+        parts = [c for c in self.chunks
+                 if _PIECE_ID.sub("", c.get("id", "")) == base
+                 and c.get("lang", "en") == first_lang]
+        return sorted(parts, key=_piece_no)
+
     def _build_xref(self):
         """Index legislation by (act, section); find each Act's definitions
         section; link every regulation to its enabling Act and every

canlex/rehab.py

@@ -17,7 +17,19 @@ Pure rule logic, verified against the corpus text of IRPR ss. 17, 18 and 18.1
   Individual rehabilitation (IRPA s. 36(3)(c), period per IRPR s. 17):
   eligible to APPLY 5 years after sentence completion (conviction branches)
   or 5 years after commission (act branches). Available for serious
-  criminality too.
+  criminality too. BUT s. 17 prescribes that period only "if the person has
+  not been convicted of a subsequent offence" (other than a contravention or
+  a YCJA matter): a later conviction restarts the clock from completion of
+  ITS sentence. The tool cannot tell whether the other matters on a record
+  pre- or post-date the one being screened, so whenever the record is not
+  clean it reports 'apply-conditional' and says what decides it, rather
+  than affirming an eligibility s. 17 may deny.
+
+  Act branch, summary-only acts: IRPA s. 36(1)(c)/(2)(c) reach only an act
+  that would be an INDICTABLE offence in Canada. There is no two-summary-acts
+  rule -- that rule exists only for CONVICTIONS (s. 36(2)(a)/(b)) -- so two
+  or more summary-only committed acts ground no criminal inadmissibility and
+  there is nothing to rehabilitate.
 
   In-Canada convictions are outside the rehabilitation scheme entirely --
   the cure is a record suspension (IRPA s. 36(3)(b)) -- EXCEPT the IRPR
@@ -49,13 +61,41 @@ def assess(basis, max_penalty_lt_10, count, years_elapsed, clean_record):
         "recent summary convictions; at most one older summary conviction; no "
         "other committed acts) -- confirm each condition against the full "
         "record before relying on deemed rehabilitation.",
-        "A temporary resident permit (IRPA s. 24) remains available at any "
-        "time regardless of rehabilitation status.",
+        "A temporary resident permit (IRPA s. 24) may be available to a "
+        "foreign national regardless of rehabilitation status -- at the "
+        "officer's discretion and subject to the s. 24(3.1)-(5) bars (e.g. a "
+        "failed refugee claimant within 12 months of the claim's rejection).",
         "Time runs from the day AFTER completion of the ENTIRE sentence -- "
         "including probation, fines and driving prohibitions -- not from the "
         "conviction date.",
+        "IRPR s. 17 prescribes the five-year application period only 'if the "
+        "person has not been convicted of a subsequent offence' (other than a "
+        "contravention or a Youth Criminal Justice Act matter): where a later "
+        "conviction exists, the five years run from completion of the "
+        "sentence for the MOST RECENT conviction, not from the matter being "
+        "screened.",
     ]
 
+    def _application(provision, lead):
+        """Eligibility to apply under s. 17 -- which a subsequent conviction
+        defeats. The record is described by one bool, so a dirty record is
+        reported as conditional and the deciding fact is named."""
+        if clean_record:
+            return _r("eligible-to-apply", provision,
+                      f"{lead} at least five years have passed since {since}, "
+                      "so the person is eligible to APPLY for individual "
+                      "rehabilitation.", caveats)
+        return _r("apply-conditional", provision,
+                  f"{lead} five years have passed since {since} -- but the "
+                  "record is not clean, and IRPR s. 17 prescribes the "
+                  "five-year period only 'if the person has not been "
+                  "convicted of a subsequent offence'. If any other "
+                  "conviction POSTDATES this matter, eligibility to apply "
+                  "runs from completion of the sentence for the MOST RECENT "
+                  "conviction: recompute years_elapsed from that date. If "
+                  "every other matter PREDATES this one, the person is "
+                  "eligible to apply now.", caveats)
+
     if basis == "canada-conviction":
         if count == "two-plus-summary-only":
             if years_elapsed >= 5:
@@ -78,16 +118,26 @@ def assess(basis, max_penalty_lt_10, count, years_elapsed, clean_record):
     since = ("sentence completion" if basis == "foreign-conviction"
              else "the commission of the act")
 
+    if basis == "foreign-act" and count == "two-plus-summary-only":
+        return _r("not-inadmissible", "IRPA s. 36(2)(c)",
+                  "A committed act grounds inadmissibility only where it "
+                  "would be an INDICTABLE offence in Canada (IRPA "
+                  "s. 36(1)(c)/(2)(c)); there is no two-summary-acts rule -- "
+                  "that rule exists only for CONVICTIONS (s. 36(2)(a)/(b)). "
+                  "Two or more summary-only committed acts ground no "
+                  "criminal inadmissibility, so there is nothing to "
+                  "rehabilitate. Verify that none of the acts has an "
+                  "indictable (including hybrid) Canadian equivalent.",
+                  caveats)
+
     if not max_penalty_lt_10:
         # Serious criminality: deemed rehabilitation is never available.
         if years_elapsed >= 5:
-            return _r("eligible-to-apply", "IRPA s. 36(3)(c); IRPR s. 17",
-                      "The equivalent Canadian offence carries a maximum of "
-                      "10 years or more (serious criminality), so deemed "
-                      "rehabilitation is NOT available -- but at least five "
-                      f"years have passed since {since}, so the person is "
-                      "eligible to APPLY for individual rehabilitation.",
-                      caveats)
+            return _application(
+                "IRPA s. 36(3)(c); IRPR s. 17",
+                "The equivalent Canadian offence carries a maximum of 10 "
+                "years or more (serious criminality), so deemed "
+                "rehabilitation is NOT available --")
         return _r("not-yet", "IRPR s. 17",
                   "Serious criminality: deemed rehabilitation is not "
                   "available, and the five-year eligibility period to apply "
@@ -96,12 +146,11 @@ def assess(basis, max_penalty_lt_10, count, years_elapsed, clean_record):
 
     if count == "multiple":
         if years_elapsed >= 5:
-            return _r("eligible-to-apply", "IRPA s. 36(3)(c); IRPR s. 17",
-                      "More than one indictable-equivalent conviction/act: "
-                      "deemed rehabilitation is not available (IRPR s. 18(2) "
-                      "covers only a single such conviction or act), but the "
-                      "five-year period to apply for individual "
-                      "rehabilitation has run.", caveats)
+            return _application(
+                "IRPA s. 36(3)(c); IRPR s. 17",
+                "More than one indictable-equivalent conviction/act: deemed "
+                "rehabilitation is not available (IRPR s. 18(2) covers only "
+                "a single such conviction or act), but")
         return _r("not-yet", "IRPR s. 17",
                   "Multiple indictable-equivalent matters: only individual "
                   "rehabilitation is possible, and its five-year eligibility "
@@ -128,10 +177,9 @@ def assess(basis, max_penalty_lt_10, count, years_elapsed, clean_record):
         why = (f"only {years_elapsed} of {need} years have run"
                if years_elapsed < need else
                "the record is not clean, which defeats deemed rehabilitation")
-        return _r("eligible-to-apply", f"{para}; IRPR s. 17",
-                  f"Deemed rehabilitation is not (yet) made out ({why}), but "
-                  "five years have passed, so the person is eligible to APPLY "
-                  "for individual rehabilitation.", caveats)
+        return _application(
+            f"{para}; IRPR s. 17",
+            f"Deemed rehabilitation is not (yet) made out ({why}), but")
     return _r("not-yet", f"{para}; IRPR s. 17",
               f"Neither deemed rehabilitation ({years_elapsed} of {need} "
               "years) nor application eligibility (five years) has been "

canlex/server.py

@@ -137,6 +137,78 @@ def _guidance_age_note(current_to: str) -> str:
     return ""
 
 
+# The '(part k of n)' / '(part k)' marker the ingesters append to a split
+# section's citation or title. Stripped wherever a whole section is served
+# or a pin-cite is built, so the client never emits 'IRPA, s. 36 (part 1 of
+# 2)(1)(c)' as a citation.
+_PART_NOTE = re.compile(
+    r"\s*\((?:part|partie)\s+\d+(?:\s+(?:of|de)\s+\d+)?\)", re.IGNORECASE)
+
+
+def _clean_citation(citation: str) -> str:
+    return _PART_NOTE.sub("", citation)
+
+
+def _assemble(parts):
+    """One chunk carrying the whole section. Long provisions are stored as
+    '-p1', '-p2', ... pieces (IRPA s. 36 is two, IRPR s. 18 three, the CDSA's
+    Schedule I ten) and get_section used to serve only the first piece -- so
+    IRPA s. 36(3), the hybrid-deeming and rehabilitation rules the screening
+    tools point the client at, was unreachable by any tool."""
+    if len(parts) == 1:
+        return parts[0]
+    whole = dict(parts[0])
+    whole["citation"] = _clean_citation(parts[0]["citation"])
+    whole["marginal_note"] = _PART_NOTE.sub("", parts[0]["marginal_note"])
+    whole["text"] = "\n\n".join(p["text"] for p in parts)
+    whole["assembled_parts"] = len(parts)
+    whole.pop("highlight", None)
+    return whole
+
+
+def _not_found(index, act: str, section: str) -> str:
+    """A get_section/citing miss: name the closest instruments, not the whole
+    inventory (~325 names, 195 of them decisions get_section cannot serve --
+    the old dump cost ~1.1k tokens per miss and taught the client that
+    'Baker' is an Act)."""
+    import difflib
+    known = {}
+    for c in index.chunks:
+        if c.get("doc_type", "legislation") == "legislation":
+            known.setdefault(c["act_short"], c.get("act_name", ""))
+    act_lc = act.strip().lower()
+    loaded = any(act_lc in (short.lower(), name.lower())
+                 for short, name in known.items()) or any(
+        act_lc == c["act_code"].lower() for c in index.chunks)
+    if loaded:
+        return (f"Error: '{act}' is loaded but has no section '{section}'. "
+                "Check the number (a subsection form like '36(1)(a)' is "
+                "fine -- the whole section is returned), or call "
+                f"canlex_toc(act='{act}') to list its sections.")
+    # Whole-word overlap first, character similarity second: 'Immigration
+    # Act' must find IRPA and the Immigration Division Rules, not whichever
+    # short names happen to share letters with it.
+    stop = {"act", "the", "and", "of", "on", "regulations", "regs", "rules"}
+    q_words = {w for w in re.findall(r"[a-z0-9]+", act_lc) if w not in stop}
+    scored = []
+    for short, name in known.items():
+        words = {w for w in re.findall(r"[a-z0-9]+", f"{short} {name}".lower())
+                 if w not in stop}
+        overlap = (len(q_words & words) / len(q_words)) if q_words else 0.0
+        ratio = max(difflib.SequenceMatcher(None, act_lc, short.lower()).ratio(),
+                    difflib.SequenceMatcher(None, act_lc, name.lower()).ratio())
+        score = overlap + ratio
+        if score >= 0.8:
+            scored.append((score, short))
+    shorts = [short for _, short in sorted(scored, reverse=True)[:3]]
+    hint = (" Did you mean: " + "; ".join(
+        f"{sh} ({known[sh]})" if known[sh] else sh for sh in shorts) + "?"
+        if shorts else "")
+    return (f"Error: no loaded instrument is named '{act}'.{hint} "
+            "canlex_list_acts gives the full inventory; "
+            "canlex_search_legislation finds a provision by topic.")
+
+
 def _format_section(c: dict, related=None) -> str:
     """Render one chunk (legislation, D-Memo, or agreement) as cited Markdown."""
     doc_type = c.get("doc_type", "legislation")
@@ -145,6 +217,11 @@ def _format_section(c: dict, related=None) -> str:
     lines = [header]
     if location:
         lines.append(f"*{location}*")
+    if c.get("assembled_parts", 0) > 1:
+        lines.append(f"_Complete text, assembled from "
+                     f"{c['assembled_parts']} stored parts._")
+    if c.get("applicability"):
+        lines.append(f"_**⚠ APPLICABILITY:** {c['applicability']}_")
     if doc_type == "memorandum":
         lines.append("_CBSA administrative guidance — persuasive, not binding law._")
         lines.append(f"(modified {_stated(c['current_to'])})")
@@ -245,7 +322,7 @@ def _format_section(c: dict, related=None) -> str:
     if hl:
         label, snippet = hl
         lines.append(f"**Most on point for this query:** "
-                     f"{c['citation']}{label} — {snippet}")
+                     f"{_clean_citation(c['citation'])}{label} — {snippet}")
     lines.append("")
     lines.append(c["text"])
     lines.append("")
@@ -277,7 +354,7 @@ def _format_section(c: dict, related=None) -> str:
     return "\n".join(lines)
 
 
-# The eight doc_type values, as a Literal so every client schema carries them
+# The nine doc_type values, as a Literal so every client schema carries them
 # as an enum rather than free text (regex 'pattern' constraints are dropped by
 # several function-calling translators, notably Gemini's).
 DocType = Literal["legislation", "memorandum", "agreement", "directive",
@@ -293,8 +370,9 @@ _SECTION_PREFIX = re.compile(
 def _normalize_section(section: str) -> str:
     """Accept the forms other tool outputs teach a client to write -- 's. 34',
     'section 34', '34(1)(c)' -- and reduce them to the bare section number the
-    index keys on. Subsection parentheticals are truncated (the full section
-    text contains its subsections anyway)."""
+    index keys on. Subsection parentheticals are truncated: the section is
+    served whole (every stored part, see _assemble), so '36(3)(a)' resolves
+    to the text that contains subsection (3)."""
     s = _SECTION_PREFIX.sub("", section.strip())
     s = s.split("(", 1)[0].strip().rstrip(".")
     return s or section.strip()
@@ -320,9 +398,11 @@ def canlex_search_legislation(
         "and operational guidance), 'agreement' (collective agreements), "
         "'directive' (NJC directives), 'caselaw' (court and tribunal "
         "decisions), 'delegation' (IRPA/IRPR delegation instruments), "
-        "'benefits' (benefit-plan member booklets), or 'commentary' "
+        "'benefits' (benefit-plan member booklets), 'commentary' "
         "(CanLex's curated analysis datasets, e.g. US dispositions vs the "
-        "IRPA 'conviction' concept). Omit to search all.")] = None,
+        "IRPA 'conviction' concept), or 'union' (CIU/PSAC by-laws, "
+        "regulations, branch rules, member guides and site VSSAs). Omit to "
+        "search all.")] = None,
     court: Annotated[Optional[str], Field(
         description="Optional: restrict CASE-LAW results to a deciding "
         "court/tribunal by name fragment, e.g. 'Supreme', 'Federal Court "
@@ -425,15 +505,13 @@ def canlex_get_section(
         - "What does subsection 20.1 say" -> act='IRPA', section='20.1'.
     """
     index = _index()
-    result = index.get_section(act, section)
-    if result is None and section != _normalize_section(section):
-        result = index.get_section(act, _normalize_section(section))
-    if result is None:
-        acts = sorted({c["act_short"] for c in index.chunks})
-        return (f"Error: no section '{section}' found in '{act}'. "
-                f"Loaded Acts: {', '.join(acts) or 'none'}. Check the section number, "
-                f"or use canlex_search_legislation to locate the provision by topic.")
-    return GROUNDING_NOTE + "\n\n" + _format_section(result, index.related(result))
+    parts = index.get_section_parts(act, section)
+    if not parts and section != _normalize_section(section):
+        parts = index.get_section_parts(act, _normalize_section(section))
+    if not parts:
+        return _not_found(index, act, section)
+    return (GROUNDING_NOTE + "\n\n"
+            + _format_section(_assemble(parts), index.related(parts[0])))
 
 
 def _load_dispositions():
@@ -950,10 +1028,9 @@ def canlex_citing(
     if result is None and section != _normalize_section(section):
         result = index.get_section(act, _normalize_section(section))
     if result is None:
-        return (f"No section '{section}' in '{act}'. Use canlex_list_acts "
-                f"for the loaded instruments.")
+        return _not_found(index, act, section)
     related = index.related(result) or {}
-    lines = [f"# Material citing {result['citation']}", ""]
+    lines = [f"# Material citing {_clean_citation(result['citation'])}", ""]
     memos = related.get("memoranda")
     if memos:
         lines.append("**Guidance citing this section** (persuasive, not "
@@ -1169,10 +1246,14 @@ def us_record_screening(disposition: str, state: str = "",
         "offence does it equate to, and is the branch serious criminality "
         "(s. 36(1)(b)) or criminality (s. 36(2)(b))? Note the maximum "
         "penalty — whether it is under 10 years feeds step 3.\n"
-        "STEP 3 — canlex_rehabilitation: compute full years since sentence "
-        "completion from today's date, pass basis='foreign-conviction', the "
-        "max-penalty answer from step 2, the count, the elapsed years, and "
-        "whether the record is otherwise clean.\n\n"
+        "STEP 3 — canlex_rehabilitation: if step 1 found a conviction (YES "
+        "or LIKELY YES), pass basis='foreign-conviction' with full years "
+        "since completion of the ENTIRE sentence, computed from today's "
+        "date; if step 1 found NO conviction and the act branch is in play, "
+        "pass basis='foreign-act' with full years since the act was "
+        "COMMITTED. Pass the max-penalty answer from step 2, the count, and "
+        "whether the record is otherwise clean (any later conviction makes "
+        "eligibility to apply conditional -- IRPR s. 17).\n\n"
         "Then give a single bottom-line assessment with every authority "
         "cited, every uncertainty flagged, and the reminder that this is "
         "legal information, not legal advice, and that an officer's "
@@ -1221,6 +1302,88 @@ def _res_equivalency_framework() -> str:
     return "The equivalency dataset is not present in this deployment."
 
 
+def _utf8_body(body: bytes):
+    """(body, changed): the request body re-encoded to UTF-8 when it was not.
+
+    The Space's run log showed a real client posting a Latin-1 body -- 'é'
+    as 0xE9 -- and the MCP transport answering HTTP 500 with a
+    UnicodeDecodeError, i.e. every accented (French) query from that client
+    failed. JSON-RPC bodies are text, so a body that is not valid UTF-8 is
+    re-read as cp1252 (Latin-1's superset on Windows clients) and re-encoded.
+    """
+    try:
+        body.decode("utf-8")
+        return body, False
+    except UnicodeDecodeError:
+        pass
+    for enc in ("cp1252", "latin-1"):
+        try:
+            return body.decode(enc).encode("utf-8"), True
+        except UnicodeDecodeError:
+            continue
+    return body, False
+
+
+_seen_clients = set()
+
+
+def _note_client(body: bytes) -> None:
+    """One stderr line per distinct MCP client (name + version) at
+    initialize -- the only way to tell which client sent a malformed body."""
+    if b'"initialize"' not in body:
+        return
+    try:
+        info = json.loads(body).get("params", {}).get("clientInfo", {})
+    except Exception:
+        return
+    tag = f"{info.get('name', '?')} {info.get('version', '')}".strip()
+    if tag not in _seen_clients:
+        _seen_clients.add(tag)
+        print(f"CanLex MCP: client {tag}", file=sys.stderr)
+
+
+class _Utf8Body:
+    """ASGI wrapper around the streamable-HTTP app: buffers each POST body,
+    re-encodes it to UTF-8 if needed (see _utf8_body) and logs the client at
+    initialize. Non-POST requests pass straight through."""
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http" or scope.get("method") != "POST":
+            await self.app(scope, receive, send)
+            return
+        chunks, queued = [], []
+        while True:
+            message = await receive()
+            if message["type"] != "http.request":
+                queued.append(message)          # e.g. http.disconnect
+                break
+            chunks.append(message.get("body", b""))
+            if not message.get("more_body"):
+                break
+        body, changed = _utf8_body(b"".join(chunks))
+        if changed:
+            scope = dict(scope)
+            scope["headers"] = [
+                (k, v) for k, v in scope.get("headers", [])
+                if k.lower() != b"content-length"
+            ] + [(b"content-length", str(len(body)).encode())]
+            print("CanLex MCP: re-encoded a non-UTF-8 request body as UTF-8",
+                  file=sys.stderr)
+        _note_client(body)
+        replay = [{"type": "http.request", "body": body,
+                   "more_body": False}] + queued
+
+        async def receive_replay():
+            if replay:
+                return replay.pop(0)
+            return await receive()
+
+        await self.app(scope, receive_replay, send)
+
+
 if __name__ == "__main__":
     try:
         idx = _index()
@@ -1231,7 +1394,10 @@ if __name__ == "__main__":
         sys.exit(1)
     # CANLEX_HTTP switches on the remote (web) transport; default is local stdio.
     if os.environ.get("CANLEX_HTTP"):
+        import uvicorn
         print("CanLex MCP: serving over streamable-HTTP.", file=sys.stderr)
-        mcp.run(transport="streamable-http")
+        uvicorn.run(_Utf8Body(mcp.streamable_http_app()),
+                    host=mcp.settings.host, port=mcp.settings.port,
+                    log_level=mcp.settings.log_level.lower())
     else:
         mcp.run()

canlex/synonyms.py

@@ -50,6 +50,17 @@ _SYNONYMS = [
     (r"danger opinion", "danger to the public refoulement protection"),
     (r"ministerial relief", "national interest exception inadmissibility"),
     (r"poe", "port of entry"),
+    # Device examinations: officers say 'cell phone' / 'laptop'; Customs Act
+    # s. 99 says 'goods' and ENF 12 s. 6.15 'digital devices'. Without the
+    # bridge an arrest-context SCC case (Fearon) and two unrelated 'without
+    # warrant' marginal notes outrank the governing border authorities.
+    (r"(?:cell ?phones?|smart ?phones?|mobile (?:phones?|devices?)|laptops?"
+     r"|tablets?|digital devices?|electronic devices?)",
+     "examination of goods digital device electronic device"),
+    # Money: 'cash', 'USD', '10k' never reach PCMLTFA s. 12's own words
+    # ('currency or monetary instruments ... prescribed amount').
+    (r"(?:cash|us ?dollars|usd|10 ?k|ten thousand dollars)",
+     "currency or monetary instruments report prescribed amount"),
     (r"flagpol(?:e|ing)", "leaving canada and reentering port of entry"),
     # The single most common US-screening term: bridge DUI shorthand to the
     # Criminal Code s. 320.14 vocabulary the equivalency analysis turns on.

data/eval/questions.json

@@ -457,6 +457,10 @@
    [
     "Canada Labour Code",
     "169"
+   ],
+   [
+    "FB Agreement",
+    "25"
    ]
   ],
   "holdout": true
@@ -888,6 +892,10 @@
    [
     "Canada Labour Code",
     "239"
+   ],
+   [
+    "FB Agreement",
+    "35"
    ]
   ]
  },
@@ -2674,5 +2682,71 @@
    ]
   ],
   "style": "french"
+ },
+ {
+  "query": "can an officer search a traveller's cell phone at the port of entry without a warrant",
+  "answers": [
+   [
+    "Canfield",
+    ""
+   ],
+   [
+    "Pike",
+    ""
+   ],
+   [
+    "ENF 12",
+    "ENF 12 s. 6.15"
+   ],
+   [
+    "Customs Act",
+    "99"
+   ]
+  ],
+  "style": "fact-pattern"
+ },
+ {
+  "query": "how much notice does the employer have to give before changing my shift schedule, and what is the premium if less",
+  "answers": [
+   [
+    "FB Agreement",
+    "25"
+   ]
+  ],
+  "style": "standard"
+ },
+ {
+  "query": "time limit to present a grievance at the first level under the FB collective agreement",
+  "answers": [
+   [
+    "FB Agreement",
+    "18"
+   ]
+  ],
+  "style": "standard"
+ },
+ {
+  "query": "traveller has USD 9,500 in cash, does it have to be reported and how is the CAD 10,000 threshold converted",
+  "answers": [
+   [
+    "PCMLTFA",
+    "12"
+   ],
+   [
+    "Cross-border Currency Reporting Regs",
+    "2"
+   ]
+  ],
+  "style": "fact-pattern"
+ },
+ {
+  "query": "refugee claimant at the land border arriving from the US: Safe Third Country exceptions for family members",
+  "answers": [
+   [
+    "IRPR",
+    "159.5"
+   ]
+  ],
+  "style": "fact-pattern"
  }
-]
+]

tests/test_rehab.py

@@ -19,10 +19,26 @@ class DeemedRehabTests(unittest.TestCase):
         r = assess("foreign-conviction", True, "one", 9, True)
         self.assertEqual(r["status"], "eligible-to-apply")
 
-    def test_dirty_record_defeats_deemed(self):
+    def test_dirty_record_defeats_deemed_and_makes_applying_conditional(self):
+        # IRPR s. 17: the five-year period runs only 'if the person has not
+        # been convicted of a subsequent offence' -- a dirty record cannot be
+        # reported as a clean 'eligible to apply'.
         r = assess("foreign-conviction", True, "one", 12, False)
+        self.assertEqual(r["status"], "apply-conditional")
+        self.assertIn("subsequent offence", r["explanation"])
+        self.assertTrue(any("subsequent offence" in c for c in r["caveats"]))
+
+    def test_clean_record_short_of_deemed_is_plainly_eligible(self):
+        r = assess("foreign-conviction", True, "one", 7, True)
         self.assertEqual(r["status"], "eligible-to-apply")
 
+    def test_two_summary_only_acts_are_not_inadmissible(self):
+        # s. 36(2)(c) needs an indictable-equivalent act; the two-summary
+        # rule exists only for convictions.
+        r = assess("foreign-act", True, "two-plus-summary-only", 2, True)
+        self.assertEqual(r["status"], "not-inadmissible")
+        self.assertIn("36(2)(c)", r["provision"])
+
     def test_two_summary_convictions_deemed_after_5_years(self):
         r = assess("foreign-conviction", True, "two-plus-summary-only", 5, True)
         self.assertEqual(r["status"], "deemed-rehabilitated")
@@ -40,6 +56,11 @@ class SeriousCriminalityTests(unittest.TestCase):
         self.assertEqual(r["status"], "eligible-to-apply")
         self.assertIn("17", r["provision"])
 
+    def test_serious_criminality_dirty_record_is_conditional(self):
+        r = assess("foreign-conviction", False, "one", 30, False)
+        self.assertEqual(r["status"], "apply-conditional")
+        self.assertIn("MOST RECENT", r["explanation"])
+
     def test_under_5_years_nothing_available(self):
         r = assess("foreign-conviction", False, "one", 3, True)
         self.assertEqual(r["status"], "not-yet")

tests/test_server.py

@@ -0,0 +1,146 @@
+"""Unit tests for the MCP tool layer (canlex/server.py).
+
+Offline: the tools are exercised against a stub LegislationIndex built via
+__new__ (no corpus, embeddings or reranker), and the ASGI body wrapper is
+driven with hand-made scope/receive callables.
+
+    python -m unittest discover -s tests
+"""
+import asyncio
+import unittest
+
+from canlex import server
+from canlex.index import LegislationIndex
+
+
+def chunk(id, section, text, marginal_note="Title", citation=None,
+          act_short="IRPA", act_code="I-2.5",
+          act_name="Immigration and Refugee Protection Act", **extra):
+    c = {"id": id, "doc_type": "legislation", "act_code": act_code,
+         "act_short": act_short, "act_name": act_name, "section": section,
+         "marginal_note": marginal_note, "part": "", "division": "",
+         "heading": "", "text": text, "history": "", "last_amended": "",
+         "in_force": "", "status": "in force", "current_to": "2026-03-31",
+         "citation": citation or f"{act_short}, s. {section}",
+         "source_url": "https://example.test"}
+    c.update(extra)
+    return c
+
+
+def stub_index(chunks):
+    idx = LegislationIndex.__new__(LegislationIndex)
+    idx.chunks = chunks
+    idx.related = lambda c: {}
+    return idx
+
+
+class GetSectionTests(unittest.TestCase):
+    def setUp(self):
+        server._INDEX = stub_index([
+            chunk("I-2.5-s35", "35", "35 (1) A permanent resident ..."),
+            chunk("I-2.5-s36-p1", "36",
+                  "36 (1) A permanent resident or a foreign national is "
+                  "inadmissible on grounds of serious criminality ...",
+                  marginal_note="Serious criminality",
+                  citation="IRPA, s. 36 (part 1 of 2)"),
+            chunk("I-2.5-s36-p2", "36",
+                  "(3) The following provisions govern subsections (1) and (2): "
+                  "(a) an offence that may be prosecuted either summarily or by "
+                  "way of indictment is deemed to be an indictable offence ...",
+                  marginal_note="Serious criminality",
+                  citation="IRPA, s. 36 (part 2 of 2)"),
+        ])
+
+    def tearDown(self):
+        server._INDEX = None
+
+    def test_every_stored_part_is_served(self):
+        out = server.canlex_get_section("IRPA", "36(3)(a)")
+        self.assertIn("(3) The following provisions govern", out)
+        self.assertIn("assembled from 2 stored parts", out)
+        self.assertIn("### IRPA, s. 36 — Serious criminality", out)
+        self.assertNotIn("(part 1 of 2)", out)
+
+    def test_unsplit_section_is_unchanged(self):
+        out = server.canlex_get_section("IRPA", "s. 35")
+        self.assertIn("### IRPA, s. 35", out)
+        self.assertNotIn("assembled", out)
+
+    def test_unknown_act_gets_closest_match_not_the_inventory(self):
+        out = server.canlex_get_section("Immigration Act", "34")
+        self.assertIn("Did you mean", out)
+        self.assertIn("IRPA", out)
+        self.assertNotIn("Loaded Acts:", out)
+
+    def test_known_act_missing_section_says_so(self):
+        out = server.canlex_get_section("IRPA", "999")
+        self.assertIn("is loaded but has no section '999'", out)
+        self.assertIn("canlex_toc", out)
+
+
+class CitationCleaningTests(unittest.TestCase):
+    def test_part_marker_is_stripped(self):
+        self.assertEqual(server._clean_citation("IRPA, s. 36 (part 1 of 2)"),
+                         "IRPA, s. 36")
+        self.assertEqual(server._clean_citation("IRPA, art. 36 (partie 2 de 2)"),
+                         "IRPA, art. 36")
+        self.assertEqual(server._clean_citation("FB Agreement, art. 25"),
+                         "FB Agreement, art. 25")
+
+
+def _run(app, scope, messages):
+    seen = {}
+
+    async def inner(scope, receive, send):
+        msg = await receive()
+        seen["body"] = msg["body"]
+        seen["headers"] = dict(scope.get("headers", []))
+        seen["next"] = await receive()
+
+    async def receive():
+        return messages.pop(0)
+
+    asyncio.run(app(inner)(scope, receive, None))
+    return seen
+
+
+class Utf8BodyTests(unittest.TestCase):
+    def test_latin1_body_is_reencoded_and_length_fixed(self):
+        body = "détention à la frontière".encode("latin-1")
+        scope = {"type": "http", "method": "POST",
+                 "headers": [(b"content-length", str(len(body)).encode())]}
+        seen = _run(server._Utf8Body, scope, [
+            {"type": "http.request", "body": body, "more_body": False},
+            {"type": "http.disconnect"}])
+        text = seen["body"].decode("utf-8")
+        self.assertEqual(text, "détention à la frontière")
+        self.assertEqual(seen["headers"][b"content-length"],
+                         str(len(text.encode("utf-8"))).encode())
+        self.assertEqual(seen["next"]["type"], "http.disconnect")
+
+    def test_utf8_body_passes_untouched_across_chunks(self):
+        body = "détention".encode("utf-8")
+        scope = {"type": "http", "method": "POST",
+                 "headers": [(b"content-length", str(len(body)).encode())]}
+        seen = _run(server._Utf8Body, scope, [
+            {"type": "http.request", "body": body[:3], "more_body": True},
+            {"type": "http.request", "body": body[3:], "more_body": False},
+            {"type": "http.disconnect"}])
+        self.assertEqual(seen["body"], body)
+
+    def test_get_requests_pass_straight_through(self):
+        calls = []
+
+        async def inner(scope, receive, send):
+            calls.append(receive)
+
+        async def receive():
+            raise AssertionError("must not be read by the wrapper")
+
+        asyncio.run(server._Utf8Body(inner)(
+            {"type": "http", "method": "GET", "headers": []}, receive, None))
+        self.assertIs(calls[0], receive)
+
+
+if __name__ == "__main__":
+    unittest.main()