diff --git "a/canlex/screening.py" "b/canlex/screening.py" new file mode 100644--- /dev/null +++ "b/canlex/screening.py" @@ -0,0 +1,6735 @@ +"""The s. 36 screening chain's shared logic: offence matching, Tran timing, +the structured hand-off block, and the one-call screening memo. + +Pure functions over the curated datasets, so every rule here is unit-tested +without a model or an index. The MCP tools in server.py are thin wrappers. + +Why a matcher this careful: the first version scored a pairing by the share +of ITS term's words found in the query, so 'reckless driving' landed on DUI, +'sexual assault' on simple assault and 'minor in possession of alcohol' on +cannabis possession -- each printed under a 'verified' heading. A wrong +pairing is worse than none, so this matcher refuses to guess: a pairing is +returned only when the query is covered by one of its terms (or a statute +alias resolves to it), negative terms veto it, and anything less confident is +offered as a "closest by meaning" suggestion from the embedder, labelled as +such, or not at all. Ties between two pairings are broken by the longest +matching term ('sexual battery' beats 'battery'); a tie that survives that +is reported as a tie, never silently resolved. +""" +import contextlib +import contextvars +import copy +import datetime as _dt +import functools +import json +import math +import re +import sys +import threading +import unicodedata + +# Words that carry no offence identity; ignored when comparing a query to a +# pairing's terms. 'attempt' is deliberately NOT here (see attempt_note). +STOP = frozenset("""a an the of in on at with and or to for by from as is was +charge charged charges conviction convicted convictions offence offense +offenses offences misdemeanor misdemeanour felony felonies count counts +class degree first second third fourth fifth 1st 2nd 3rd 4th 5th state +federal us usa u s crime criminal record plea pled guilty while under without +against upon over ct cts ucmj usc ipc rcw mcl vtl nrs ors article section +statute code title chapter para paragraph subsection""".split()) + +# Rap-sheet shorthand, expanded before matching. Whole tokens only; a +# statute alias or a pairing term is matched on the expanded text. +ABBREVIATIONS = { + "poss": "possession", "poss.": "possession", "possess": "possession", + "w/": "with", "w/o": "without", "agg": "aggravated", "aggr": "aggravated", + "att": "attempted", "attempt": "attempted", "cds": "controlled substance", + "cs": "controlled substance", "dv": "domestic violence", + "mv": "motor vehicle", "veh": "vehicle", "deg": "degree", + "pwid": "possession with intent to distribute", + "pwitd": "possession with intent to distribute", + "pwisd": "possession with intent to sell or deliver", + "dwls": "driving while license suspended", + "dwlr": "driving while license revoked", + "dwlsr": "driving while license suspended or revoked", + "dus": "driving under suspension", "dwai": "driving while ability impaired", + "ovi": "operating a vehicle impaired", "owvi": "operating while visibly impaired", + "owpd": "operating with presence of drugs", + "ocs": "operating a vehicle under the influence", + "udaa": "unlawfully driving away an automobile", + "r&o": "resisting and obstructing", "r&c": "resisting and obstructing", + "mdop": "malicious destruction of property", + "a&b": "assault and battery", "abdw": "assault and battery with a dangerous weapon", + "adw": "assault with a deadly weapon", "oas": "operating after suspension", + "cpw": "criminal possession of a weapon", + "cpcs": "criminal possession of a controlled substance", + "cscs": "criminal sale of a controlled substance", + "upm": "unlawful possession of marijuana", "oga": "obstructing governmental administration", + "mip": "minor in possession of alcohol", "pi": "public intoxication", + "csc": "criminal sexual conduct", "gbh": "grievous bodily harm", + "abh": "actual bodily harm", "twoc": "taking without owner's consent", + "pwits": "possession with intent to supply", "ucmj": "ucmj", + "art": "article", "arts": "article", "sec": "section", "sect": "section", + "usc": "usc", "ipc": "ipc", + # roadmap T1.3 (2026-09-12): rap-sheet shorthand that dead-ended the + # chain. Each expansion only routes a label to a pairing's own terms; the + # pairing's verified maximum is unchanged. + "aslt": "assault", "asslt": "assault", "batt": "battery", "burg": "burglary", + "burgl": "burglary", "marij": "marijuana", "paraph": "paraphernalia", + "crim": "criminal", "intox": "intoxication", "inj": "injury", + "ucw": "unlawful carrying of a weapon", "fta": "failure to appear", + "fail": "failure", "dwli": "driving while license invalid", + "ouil": "operating under the influence of liquor", + "duii": "driving under the influence of intoxicants", + "w/dw": "with a deadly weapon", "b&e": "breaking and entering", + # roadmap T1.3 gold misses (track A step A9): common misspellings on + # records. Never 'posesion' -- that is the Spanish word ('posesión de + # narcóticos' is a curated term). + "possesion": "possession", "posession": "possession", "possesson": "possession", + "controled": "controlled", +} + +# Multi-word shorthand, replaced as whole phrases on the folded text before +# the token-level table runs ('cont sub' is two tokens; 'crim tres' must not +# turn every Spanish 'tres' into trespass). +PHRASE_ABBREVIATIONS = { + "cont sub": "controlled substance", "cont subst": "controlled substance", + "crim tres": "criminal trespass", "crim tresp": "criminal trespass", + "theft prop": "theft of property", "w/ dw": "with a deadly weapon", + # 'POSS SM AMT MARIJ' (Pennsylvania's 'small amount of marihuana'); a + # phrase only -- a bare 'SM' is a misdemeanour class on some rap sheets + "sm amt": "small amount", +} + +# 'para' is rap-sheet shorthand for paraphernalia only beside a drug word: +# elsewhere it is Spanish for 'for' ('posesion de narcoticos para venta') or +# a paragraph reference, and must stay as it is. +_PARA_CONTEXT = frozenset({"drug", "drugs", "possession", "narcotic", "narcotics", + "paraphernalia"}) + +# A UCMJ article is numbered like a civil-law code article ('Article 128' is +# assault in the UCMJ and something else in many criminal codes). Bare +# 'article N' aliases and pairing terms apply only with a military cue in the +# record (critic, 2026-09-12); otherwise no alias is applied and a note says +# why. +_MILITARY_CUE = re.compile( + r"\b(ucmj|uniform code of military justice|court[- ]martial|courts[- ]martial|" + r"military|armed forces|article 15|nonjudicial punishment|njp)\b") +_UCMJ_ARTICLE = re.compile(r"^(?:article|art|arts) \d+[a-z]?$") +UCMJ_NOTE = ("'Article N' is read as a UCMJ article only when the record says " + "UCMJ, court-martial or military: civil-law criminal codes number " + "their offences as articles too. No statute alias was applied -- " + "name the code (e.g. 'UCMJ art. 128') or give the offence in words.") + +# Cannabis slang folds to 'marijuana' so the cannabis pairings' own terms +# ('possession of marijuana') are reached -- the Cannabis Act pairing, never +# the CDSA drug-possession one (critic correction to T1.3). +_CANNABIS_SLANG = re.compile(r"\b(weed|pot|thc|hashish|hash|marihuana)\b") +# Forms the Cannabis Act does not count gram for gram as dried cannabis. +_CANNABIS_CONCENTRATE = frozenset({"hash", "hashish", "thc", "wax", "shatter", + "dab", "dabs", "concentrate", "concentrates", + "resin", "rosin", "budder"}) +CANNABIS_CONCENTRATE_NOTE = ( + "CANNABIS FORM: the record names a concentrate or resin ({form}). The " + "Cannabis Act's possession limits are amounts 'equivalent to' dried " + "cannabis 'as determined in accordance with Schedule 3' (s. 8(1)(a): " + "more than 30 g of dried cannabis or its equivalent in a public place), " + "and Schedule 3 treats 0.25 g of cannabis concentrates as equivalent to " + "1 g of dried cannabis. Do not assume the amount was within the limit: " + "get the quantity and the form (C-24.5 s. 8, Schedule 3).") +CANNABIS_BARE_NOTE = ( + "The record names only the substance (cannabis): it is read as " + "possession. Confirm it was not sale, distribution or cultivation, which " + "the Cannabis Act treats differently.") +# California's attempt statute cited with the attempted offence's section +# ('PC 664/211', 'PC 664(a)/187'): read as 'attempted' plus that section, so +# the section's own alias names the offence and s. 463 applies (track A step +# A9; the 'pc 664' -> 'attempted' alias reached no pairing). A bare 'PC 664' +# names no offence and is left alone. +_PC_664 = re.compile(r"\bpc 664(?:\([a-z0-9]\))?\s*/\s*(?:pc\s*)?" + r"(\d+(?:\.\d+)?(?:\([a-z0-9]+\))*)") +# Words that make an amount or a Texas penalty group part of a drug charge. +_DRUG_CONTEXT = re.compile(r"\b(?:controlled substances?|drugs?|narcotics?)\b") +# Texas 'PG 1' / 'PG 1-A' / 'PG 2' (penalty group): the Texas schedule, which +# names neither the charge nor a CDSA schedule (gold 'POSS CS PG 1 <1G'). +_TX_PENALTY_GROUP = re.compile(r"\bpg ?(?:1 ?[ab]\b|2 ?a\b|[1-4]\b)") +PENALTY_GROUP_NOTE = ( + "PENALTY GROUP: the record names a Texas penalty group ('PG'). That is the " + "Texas schedule, not a CDSA schedule, so it does not settle the Canadian " + "maximum: give the substance, or its CDSA schedule (the schedule " + "parameter).") +# A quantity written beside a substance ('< 20G', '2 OZ') is not part of the +# charge's identity; it is dropped from the matching text only. +_QUANTITY = re.compile( + # (the code audit of 2026-09-21: a thousands separator -- '5,000 grams', or the + # comma _fold has already turned into a space, '5 000 grams' -- is part of the + # amount, so the whole amount leaves the matching text; before this the '5' stayed + # and 'sale of 5,000 grams of marijuana' reached no pairing. Its follow-up: the + # decimal _fold spaces too -- '2,5 kg' and '2.5 kg' both read '2 5 kg' here -- and a + # fraction reads '1 1 2 lbs', so a short digit run before the unit is the amount) + r"\b(?:\d{1,3}(?:[ ,]\d{3})+(?:[ .,]\d{1,2})?|(?:\d+ )?\d+ ?/ ?\d+|" + r"\d{1,3}(?:[ .,]\d{1,3}){1,2}|\d+)(?:\.\d+)?" + r"\s*(?:g|gr|gm|gms|grams?|oz|ozs|ounces?|lbs?|pounds?|kg|kgs|kilos?|kilograms?|mg|ml)\b") +# A count written beside the offence ('2 counts', 'x2', 'count 2', '3 cts') is not part +# of the charge's identity either (the audit follow-up of 2026-09-21): it leaves the +# matching text, and split_offences keeps it with its offence. +# ('2 x 2 kg bags' is a multiplier, not a count: an x beside a digit that an amount +# follows is left alone, and quantity_kg then refuses to settle the amount) +_UNIT_WORDS = r"(?:g|gr|gm|gms|grams?|oz|ozs|ounces?|lbs?|pounds?|kg|kgs|kilos?|kilograms?|mg|ml)\b" +_COUNT_WORDS = re.compile( + r"\(?\s*\b(?:x\s*\d+(?!\s*(?:[.,/]\d+)?\s*" + _UNIT_WORDS + r")|\d+\s*x(?!\s*\d)|" + r"\d+\s*(?:counts?|cts?)|counts?\s*#?\s*\d+)\b\s*\)?") + +# US street and brand names -> the substance's CDSA schedule (or cannabis). +# Verified against the corpus schedule text by tests/test_screening.py. +SUBSTANCES = { + # Schedule I + "cocaine": ("cocaine", "I"), "crack": ("cocaine", "I"), + "heroin": ("heroin", "I"), "fentanyl": ("fentanyl", "I"), + "oxycodone": ("oxycodone", "I"), "oxycontin": ("oxycodone", "I"), + "percocet": ("oxycodone", "I"), "hydrocodone": ("hydrocodone", "I"), + "vicodin": ("hydrocodone", "I"), "morphine": ("morphine", "I"), + "methadone": ("methadone", "I"), "meth": ("methamphetamine", "I"), + "methamphetamine": ("methamphetamine", "I"), + "amphetamine": ("amphetamine", "I"), "adderall": ("amphetamine", "I"), + "mdma": ("mdma", "I"), "ecstasy": ("mdma", "I"), "molly": ("mdma", "I"), + "ketamine": ("ketamine", "I"), "ghb": ("ghb", "I"), + # Schedule III + "lsd": ("lsd", "III"), "acid": ("lsd", "III"), + "psilocybin": ("psilocybin", "III"), "mushrooms": ("psilocybin", "III"), + "shrooms": ("psilocybin", "III"), "mescaline": ("mescaline", "III"), + # Schedule IV + "xanax": ("alprazolam", "IV"), "alprazolam": ("alprazolam", "IV"), + "valium": ("diazepam", "IV"), "diazepam": ("diazepam", "IV"), + "klonopin": ("clonazepam", "IV"), "clonazepam": ("clonazepam", "IV"), + "ativan": ("lorazepam", "IV"), "lorazepam": ("lorazepam", "IV"), + "ambien": ("zolpidem", "IV"), "zolpidem": ("zolpidem", "IV"), + "benzodiazepine": ("benzodiazepines", "IV"), + "benzodiazepines": ("benzodiazepines", "IV"), "benzos": ("benzodiazepines", "IV"), + "steroids": ("anabolic steroids", "IV"), + "testosterone": ("anabolic steroids", "IV"), + "barbiturates": ("barbiturates", "IV"), + # cannabis (Cannabis Act, not the CDSA) + "marijuana": ("cannabis", "cannabis"), "marihuana": ("cannabis", "cannabis"), + "cannabis": ("cannabis", "cannabis"), "weed": ("cannabis", "cannabis"), + "pot": ("cannabis", "cannabis"), "thc": ("cannabis", "cannabis"), + "hashish": ("cannabis", "cannabis"), "hash": ("cannabis", "cannabis"), +} + +# Statute codes as they appear on US records -> the plain-language charge a +# pairing term names. Matched as whole phrases (case-insensitive, punctuation +# folded) before tokenising. Curated: the curated batch of 2026-09-10 extends +# this table state by state (data/curated/us_equivalency.json "aliases" is +# merged in by load_aliases()). +ALIASES = { + # California + "vc 23152": "DUI", "23152": "DUI", "vc 23153": "DUI causing injury", + "pc 647(f)": "public intoxication", "647(f)": "public intoxication", + "647f": "public intoxication", + "pc 484": "petty theft", "pc 488": "petty theft", "pc 487": "grand theft", + "pc 459": "burglary", "pc 211": "robbery", "pc 242": "battery", + "pc 243(e)": "domestic battery", "pc 273.5": "domestic violence", + "pc 245": "assault with a deadly weapon", "pc 594": "vandalism", + "pc 148": "resisting arrest", "pc 415": "disturbing the peace", + "pc 496": "receiving stolen property", "pc 470": "forgery", + "pc 314": "indecent exposure", "pc 261": "rape", + "pc 243.4": "sexual battery", "pc 646.9": "stalking", + "pc 422": "criminal threats", "pc 25400": "carrying a concealed weapon", + "pc 29800": "felon in possession of a firearm", + "hs 11350": "drug possession", "hs 11377": "meth possession", + "hs 11357": "marijuana possession", "hs 11351": "possession for sale", + "hs 11352": "drug trafficking", "hs 11359": "marijuana possession for sale", + "hs 11360": "marijuana trafficking", "hs 11364": "drug paraphernalia", + "vc 14601": "driving while suspended", "vc 23103": "reckless driving", + "vc 20001": "hit and run", "vc 20002": "hit and run", + # Texas + "tx pc 49.04": "DWI", "pc 49.04": "DWI", "pc 49.02": "public intoxication", + "pc 22.01": "simple assault", "pc 31.03": "theft", "pc 30.02": "burglary", + "pc 42.01": "disorderly conduct", "pc 38.03": "resisting arrest", + "pc 30.05": "criminal trespass", + # New York + "vtl 1192": "DWI", "pl 240.20": "disorderly conduct", + "pl 155.25": "petit larceny", "pl 120.00": "simple assault", + "pl 220.03": "drug possession", "pl 221.05": "marijuana possession", + "pl 140.20": "burglary", "pl 160.05": "robbery", + "pl 145.00": "criminal mischief", "pl 205.30": "resisting arrest", + # Florida + "316.193": "DUI", "812.014": "theft", "784.03": "battery", + "893.13": "drug possession", "810.02": "burglary", "812.13": "robbery", + # Michigan / federal + "mcl 257.625": "OWI", "257.625": "OWI", + "8 usc 1325": "illegal entry", "8 usc 1326": "illegal re-entry", + "18 usc 1325": "illegal entry", + "18 usc 1028": "identity theft", "21 usc 844": "simple possession", + "21 usc 841": "drug trafficking", "18 usc 1343": "wire fraud", + "18 usc 1344": "bank fraud", "18 usc 922(g)": "felon in possession of a firearm", +} + +_ATTEMPT = re.compile(r"\battempt(?:ed|ing)?\b", re.I) +# A pairing is returned only at this score or above: phrase matches score +# 1.0 and drug names generalised to 'controlled substance' ~0.875, so 0.8 +# keeps every real charge and drops the partial-overlap seconds ('driving +# while suspended' ~ DUI at 0.67, 'assault on a police officer' ~ 'eluding a +# police officer' at 0.67, 'sexual assault' ~ 'felony assault' at 0.75). +MIN_SCORE = 0.8 + +# Multi-offence separators: 'DUI; possession of cocaine' is two entries and +# is screened as two, never as one string with a borrowed schedule. A comma +# between digits is a thousands separator ('fraud over $5,000'), not a +# separator between offences. +# A comma before a code designator names the code of the offence before it, +# not a second offence: 'Article 111, UCMJ' is one entry (verifier, +# 2026-09-15 -- it was split into 'Article 111' and 'UCMJ'). +_CODE_COMMA = r",(?!\d)(?!\s*(?:ucmj|u\.c\.m\.j\.?|uniform code of military justice|courts?[- ]martial)\b)" +_SPLIT = re.compile(r"\s*(?:;|" + _CODE_COMMA + r"|\band\b|&|\+|(? 1 + and out[-2].split()[-1] in _PARA_CONTEXT))): + out.append("paraphernalia") + continue + out.append(ABBREVIATIONS.get(key, ABBREVIATIONS.get(tok, tok))) + return " ".join(out) + + +def _expand_phrases(text): + for short, full in PHRASE_ABBREVIATIONS.items(): + text = re.sub(r"(?: ' there chooses the limb.""" + token = _RECORD.set((_state_name(state), _iso_day(on), facts or None)) + try: + yield + finally: + _RECORD.reset(token) + + +def load_aliases(extra): + """Merge a curated alias table (statute code -> plain charge, or an alias + object) in; used by the server at start-up so the curated dataset can + extend ALIASES without a code change. Resets the compiled alias table + and the expansion cache, which depend on it. + + A table equal to the one installed changes nothing (audit follow-up + 2026-09-21, optimization 1): the screening tools reload the curated table + on every call, and rebuilding the compiled table and clearing the + expansion cache each time cost about 17 ms and every cached expansion. + The comparison is of state -- the merged, lower-cased table against a + deep copy of the installed one, so an alias object edited in place still + counts as a change -- never of the last argument: a different table (a + test's approved baseline) is still installed, and the next load of the + curated one undoes it. + + Installing a table bumps _ALIAS_GENERATION, which keys the expansion + cache: an expansion a concurrent call computed from the old table lands + under the old generation and is never served again.""" + global _EXTRA_ALIASES, _ALIAS_TABLE, _ALIAS_GENERATION + table = {k.lower(): v for k, v in (extra or {}).items()} + with _ALIAS_LOCK: + if table == _EXTRA_ALIASES: + return + _EXTRA_ALIASES = copy.deepcopy(table) + _ALIAS_TABLE = [] + _ALIAS_GENERATION += 1 + _analyse.cache_clear() + + +def _raw_aliases(): + merged = dict(ALIASES) + merged.update(_EXTRA_ALIASES) + return merged + + +def all_aliases(): + """{code: plain charge words}: for an alias object, its top-level 'plain' + (the reading it applies when no offence date chooses another).""" + return {code: (v.get("plain") if isinstance(v, dict) else v) + for code, v in _raw_aliases().items()} + + +def alias_entries(): + """{code: the alias as an object}: a plain string becomes {"plain": ...}; + 'source' says 'curated' or 'code-table', and a code-table bare section + number carries its ALIAS_JURISDICTIONS state.""" + out = {} + for code, v in _raw_aliases().items(): + e = dict(v) if isinstance(v, dict) else {"plain": v} + curated = code in _EXTRA_ALIASES + e["source"] = "curated" if curated else "code-table" + if not curated: + juris = _code_table_jurisdiction(code) + if juris: + e.setdefault("jurisdiction", juris) + out[code] = e + return out + + +def alias_key(code): + """A statute code as a record's text is read (analyse): folded, dotted + initials closed up, shorthand expanded, Maine 'mrs' read as 'mrsa', the + UCMJ and Motor Vehicles Act forms read alike -- so '29-a m.r.s. 2411' and + 'mv act 185' can fire (verifier, 2026-09-12: the record side collapsed + 'M.R.S.' and expanded 'mv' before any alias ran).""" + text = _DOTTED.sub(lambda m: m.group(0).replace(".", ""), _fold(code)) + text = " ".join(_expand_abbreviations(_expand_phrases(text)).split()) + return _code_forms(_MAINE_MRS.sub(r"\1 mrsa", text)) + + +def alias_pattern(folded_key): + """A folded statute code as a regex with a right boundary (roadmap T1.2): + the code may be followed by the end, a space, punctuation or a + parenthesised subsection ('vc 23152(a)'), but never by a letter, a digit + or '.' -- so 'pc 459' no longer swallows 'pc 459.5' (shoplifting) + and '316.193' no longer swallows '316.1935' (fleeing or eluding). The + left boundary keeps '8 usc 1325' out of '18 usc 1325'.""" + return re.compile(r"(? 1} + for code, e in entries.items(): + key = alias_key(code) + if not key or not e.get("plain"): + continue + prev = rows.get(key) + if prev is not None and (prev[1].get("verified") or not e.get("verified")): + continue + scope = ("bare" if _BARE_KEY.match(" ".join(code.lower().split())) + else "shared" if _code_prefix(key) in shared else "prefixed") + rows[key] = (key, dict(e, key=code), alias_pattern(key), + bool(_UCMJ_ARTICLE.match(key)), scope) + _ALIAS_TABLE = sorted(rows.values(), key=lambda r: -len(r[0])) + return _ALIAS_TABLE + + +def _covers(r, on): + return (not r.get("from") or on >= r["from"]) and (not r.get("until") or on <= r["until"]) + + +def _period(r): + if not r: + return None + if r.get("from") and r.get("until"): + return f"{r['from']} to {r['until']}" + if r.get("from"): + return f"from {r['from']}" + return f"until {r['until']}" if r.get("until") else None + + +def _element_pick(r, facts): + """(plain, answer) for a reading whose limbs name different offences -- + an 'elements' split (verifier, 2026-09-15: UCMJ Art. 89(b), the former + Art. 90(1) and Art. 91(1) are assaults on a superior; the other limbs are + service offences): the limb the facts answer (': '), or + (None, None) when they answer none or more than one. A reading without a + split: (its plain, None).""" + el = r.get("elements") + if not el: + return r.get("plain"), None + by = {x["answer"]: x for x in el.get("readings") or []} + got = [a for a in (resolver_answers(el["fact"], facts) if facts else []) if a in by] + if len(got) == 1: + x = by[got[0]] + return x["plain"], {"fact": el["fact"], "answer": x["answer"], "label": x["label"]} + return None, None + + +def _alias_reading(e, key, scope, state, on, facts=None): + """How one statute code the record names is read (roadmap T3.5): the + reading for the offence date, the state rule for a code that names no + state or one several states share, and the limb the facts answer where + the reading's limbs name different offences. + {"code", "key", "status": "applied" | "other-state" | "date-open" | + "element-open" | "no-reading", "plain" (applied only), "jurisdiction", + "cite", "catchline", "cite_url", "verified", "source", "period", + "assumed", "mismatch", "scope", "readings" (every dated reading, when + none was chosen by a date), "limbs" (an element split's readings), + "fact", "question", "answer", "note", "state", "on"}. + scope: 'bare' / 'shared' codes are not applied for another state (and are + labelled 'jurisdiction assumed' without one); a 'prefixed' code names its + state, so a different record state applies it with a MISMATCH flag.""" + rds = [{k: e.get(k) for k in _READING_KEYS}] + [ + {k: r.get(k) for k in _READING_KEYS} for r in (e.get("dated") or [])] + top = rds[0] + read = {"code": key, "key": e.get("key", key), "source": e.get("source"), + "jurisdiction": e.get("jurisdiction"), "cite": e.get("cite"), + "verified": e.get("verified"), "catchline": top["catchline"], + "cite_url": top["cite_url"], "note": top["note"], "plain": None, + "period": None, "assumed": False, "mismatch": False, "scope": scope, + "readings": None, "limbs": None, "fact": None, "question": None, + "answer": None, "state": state, "on": on} + juris = (e.get("jurisdiction") or "").lower() + if juris in US_STATES.values(): + if scope in ("bare", "shared"): + if state and state != juris: + return dict(read, status="other-state") + read["assumed"] = not state + elif state and state != juris: + read["mismatch"] = True + brief = [{"period": _period(r), "plain": r["plain"], "catchline": r["catchline"], + "note": r["note"], + "limbs": [x["plain"] for x in (r.get("elements") or {}).get("readings") or []] + or None} for r in rds] + + def settle(r, **kw): + base = dict(read, catchline=r["catchline"], cite_url=r["cite_url"] or read["cite_url"], + note=r["note"], **kw) + el = r.get("elements") + if not el: + return dict(base, status="applied", plain=r["plain"]) + plain, answer = _element_pick(r, facts) + base.update(fact=el["fact"], question=el.get("question")) + if answer: + return dict(base, status="applied", plain=plain, answer=answer) + return dict(base, status="element-open", + limbs=[{"answer": x["answer"], "label": x["label"], "plain": x["plain"]} + for x in el.get("readings") or []]) + + if not any(r["from"] or r["until"] for r in rds): + return settle(top) + if on: + hit = next((r for r in rds if _covers(r, on)), None) + if hit and (hit["plain"] or hit.get("elements")): + return settle(hit, period=_period(hit)) + return dict(read, status="no-reading", catchline=(hit or {}).get("catchline"), + note=(hit or {}).get("note"), period=_period(hit), + readings=brief if hit is None else None) + whole = any(not r["from"] for r in rds) and any(not r["until"] for r in rds) + split = any(r.get("elements") for r in rds) + if whole and top["plain"] and not split and len({r["plain"] for r in rds}) == 1: + return dict(read, status="applied", plain=top["plain"], readings=brief) + return dict(read, status="date-open", readings=brief) + + +def _alias_what(r): + """' ''' for a note, without repeating a + jurisdiction the cite already names ('UCMJ art. 111').""" + juris, cite = r.get("jurisdiction") or "", r.get("cite") or "" + parts = [] + if juris and juris.split()[0].lower() not in cite.lower(): + parts.append(juris) + if cite: + parts.append(cite) + if r.get("catchline"): + parts.append(f"'{r['catchline']}'") + elif cite: + parts.append("(no official heading)") + return " ".join(parts) + + +def alias_note(r): + """The line an alias reading prints (roadmap T3.5): which statute the code + was read as and on what authority; or why it was not applied.""" + code, what = r.get("key") or r["code"], _alias_what(r) + if r["status"] == "other-state": + if r.get("scope") == "shared": + return (f"STATUTE CODE NOT APPLIED: '{code}' uses a code designator " + f"('{_code_prefix(r['code']).upper()}') that more than one state's " + f"code uses; CanLex reads this number as {what}, but the record's " + f"state is {r['state'].title()}: it was not read as that statute. " + "Give the offence in words (or the code with its own state's " + "prefix).") + return (f"STATUTE CODE NOT APPLIED: '{code}' is a bare section number CanLex " + f"reads as {what}, but the record's state is {r['state'].title()}: it " + "was not read as that statute. Give the offence in words (or the " + "code with its own state's prefix).") + if r["status"] == "element-open": + each = "; ".join(f"'{x['answer']}': {x['label']} -> '{x['plain']}'" for x in r["limbs"]) + answers = " | ".join(x["answer"] for x in r["limbs"]) + period = f" (the reading for an offence {r['period']})" if r.get("period") else "" + return (f"ELEMENT-DEPENDENT STATUTE CODE: '{code}' read as {what}{period} names " + f"more than one offence -- {each}. No alias was applied, and nothing " + f"is chosen for you: the specification decides -- answer " + f"'{r['fact']}: {answers}' in the facts (canlex_screen_record, or the " + "entry's facts in canlex_screen_history), or give the offence in words." + + (f" {r['note'].rstrip('.')}." if r.get("note") else "")) + if r["status"] == "date-open": + each = "; ".join( + f"{b['period'] or 'otherwise'}: " + + (f"'{b['catchline']}' " if b["catchline"] else "") + + ("-> " + " or ".join(f"'{p}'" for p in b["limbs"]) + " (the specification decides)" + if b.get("limbs") else f"-> '{b['plain']}'" if b["plain"] else "-> no curated pairing") + + (f" ({b['note']})" if b["note"] else "") + for b in r["readings"]) + return (f"DATE-DEPENDENT STATUTE CODE: '{code}' ({r.get('jurisdiction') or 'code'}) " + f"names a different offence depending on the offence date -- {each}. " + "No alias was applied: give offence_date (the date decides the " + "reading), or the offence in words.") + if r["status"] == "no-reading": + where = f"'{code}' ({r.get('jurisdiction') or 'code'})" + if r.get("catchline"): + why = (f"{where} is '{r['catchline']}' ({r['period']}), which no curated " + "pairing covers") + else: + why = f"no verified reading of {where} covers that date" + return (f"STATUTE CODE NOT APPLIED: at an offence date of {r['on']}, {why}" + + (f" ({r['note']})" if r.get("note") else "") + + ". Give the offence in words.") + line = (f"STATUTE CODE: '{code}' read as {what} -> '{r['plain']}'" if what + else f"STATUTE CODE: '{code}' read as '{r['plain']}'") + if r.get("period"): + line += f" (the reading for an offence {r['period']})" + elif r.get("readings"): + line += (" (the same charge in every period: " + + "; ".join(f"{b['period']}: '{b['catchline']}'" + (f" -- {b['note']}" if b["note"] else "") + for b in r["readings"] if b["catchline"]) + ")") + if r.get("answer"): + a = r["answer"] + line += f" (the facts answer '{a['fact']}: {a['answer']}': {a['label']})" + if r.get("assumed") and r.get("scope") == "shared": + line += (f". JURISDICTION ASSUMED: no state was given, and '{code}' uses a code " + f"designator more than one state's code uses, read as {r['jurisdiction']} " + f"law -- confirm the record is from {r['jurisdiction']}") + elif r.get("assumed"): + line += (f". JURISDICTION ASSUMED: no state was given, and '{code}' is a bare " + f"section number read as {r['jurisdiction']} law -- confirm the record " + f"is from {r['jurisdiction']}") + if r.get("mismatch"): + line += (f". JURISDICTION MISMATCH: '{code}' is {r['jurisdiction']} law, but the " + f"record's state is {r['state'].title()}: it was read as the " + f"{r['jurisdiction']} statute anyway -- confirm the record (the state or " + "the code is likely mis-entered) before relying on it") + if r.get("verified"): + line += (f". Verified against the official text {r['verified']}" + + (f": {r['cite_url']}" if r.get("cite_url") else "")) + elif r.get("source") == "curated": + line += ". Curated alias, not verified section by section" + else: + line += ". Code-table alias, not verified section by section" + if r.get("note") and not r.get("readings"): + line += f". {r['note']}" + return line.rstrip(".") + "." + + +_DOTTED = re.compile(r"(? + '... xanax controlled substance') and cannabis slang is read as marijuana + so the Cannabis Act pairings' own terms are reached. Pure for a given + alias table: `generation` (analyse passes _ALIAS_GENERATION) is only part + of the cache key, so an expansion is never served under a later table + (load_aliases() also clears the cache).""" + # dotted initials read as the abbreviation they spell: 'D.U.I.' -> 'dui', + # '18 U.S.C. 1343' -> '18 usc 1343', 'P.C. 459' -> 'pc 459' + text = _DOTTED.sub(lambda m: m.group(0).replace(".", ""), _fold(offense)) + text = _MAINE_MRS.sub(r"\1 mrsa", _expand_abbreviations(_expand_phrases(text))) + text = _code_forms(" ".join(text.split())) + text = " ".join(_COUNT_WORDS.sub(" ", text).split()) + notes, flags, fired, reads = [], [], [], [] + text = _PC_664.sub(lambda m: f" attempted pc {m.group(1)} ", text) + military = bool(_MILITARY_CUE.search(text)) + for key, entry, pat, ucmj, scope in _alias_table(): + if not pat.search(text): + continue + if ucmj and not military: + if "ucmj-article-ungated" not in flags: + flags.append("ucmj-article-ungated") + notes.append(UCMJ_NOTE) + continue + read = _alias_reading(entry, key, scope, state, on, facts) + reads.append(read) + notes.append(alias_note(read)) + new = [] + if read["status"] == "applied": + text = pat.sub(" " + _fold(read["plain"]).strip() + " ", text) + fired.append(key) + new += ["alias-jurisdiction-assumed"] if read["assumed"] else [] + new += ["alias-jurisdiction-mismatch"] if read["mismatch"] else [] + else: + new.append("alias-" + read["status"]) + for flag in new: + if flag not in flags: + flags.append(flag) + text = _expand_abbreviations(text) + text = re.sub(r"\s+", " ", text.replace("&", " and ")).strip() + schedule = None + words = re.findall(r"[a-z0-9]+", text) + for word in words: + if word in SUBSTANCES: + schedule = schedule or SUBSTANCES[word][1] + drug_words = bool(schedule or _DRUG_CONTEXT.search(text)) + if drug_words and _TX_PENALTY_GROUP.search(text): + text = _TX_PENALTY_GROUP.sub(" ", text) + flags.append("penalty-group") + notes.append(PENALTY_GROUP_NOTE) + if drug_words: + text = re.sub(r"\s+", " ", _QUANTITY.sub(" ", text)).strip() + if schedule == "cannabis": + forms = [w for w in words if w in _CANNABIS_CONCENTRATE] + if forms: + flags.append("cannabis-concentrate") + notes.append(CANNABIS_CONCENTRATE_NOTE.format(form=", ".join(dict.fromkeys(forms)))) + text = re.sub(r"\b(marijuana)(?: marijuana)+\b", r"\1", + _CANNABIS_SLANG.sub("marijuana", text)) + if not (content_tokens(text) - {"marijuana", "cannabis"}): + text = "possession of " + text + flags.append("cannabis-bare-substance") + notes.append(CANNABIS_BARE_NOTE) + for word in ("marijuana", "cannabis"): + if not re.search(rf"\b{word}\b", text): + text += " " + word + elif schedule: + text += " controlled substance" + return Expansion((re.sub(r"\s+", " ", text).strip(), schedule, tuple(notes), + tuple(flags), tuple(fired), tuple(reads))) + + +def expand(offense, state=None, on=None, facts=None): + """(text for matching, the substance's CDSA schedule / 'cannabis' / + None) -- see analyse().""" + a = analyse(offense, state, on, facts) + return a.text, a.schedule + + +def offence_notes(offense, state=None, on=None, facts=None): + """What the matcher noticed about the string that the officer must see + (the statute each code was read as, an ungated 'Article N', a cannabis + concentrate, a bare substance).""" + return list(analyse(offense, state, on, facts).notes) + + +def offence_flags(offense, state=None, on=None, facts=None): + return list(analyse(offense, state, on, facts).flags) + + +_READ_FIELDS = ("code", "key", "status", "plain", "jurisdiction", "cite", "catchline", + "cite_url", "verified", "source", "period", "assumed", "mismatch", + "scope", "readings", "limbs", "fact", "answer") + + +def aliases_read(offense, state=None, on=None, facts=None): + """CANLEX_RESULT 'aliases_read' (roadmap T3.5): every statute code the + record named, as read -- applied or not, and why.""" + return [{k: r.get(k) for k in _READ_FIELDS} for r in analyse(offense, state, on, facts).reads] + + +def alias_needs(offense, state=None, on=None, facts=None): + """The question a statute code leaves open: the offence date for a + date-dependent code (no offence date, readings that differ), the limb + for an element-split code the facts do not answer (parameter 'facts', + ': '). One 'needs' entry per code.""" + out = [] + for r in analyse(offense, state, on, facts).reads: + if r["status"] == "element-open": + answers = " | ".join(x["answer"] for x in r["limbs"]) + out.append({ + "fact": r["fact"], "parameter": "facts", "code": r["code"], + "question": (f"{r['question']} -- answer in the facts as " + f"'{r['fact']}: ' ({answers})"), + "options": [{"value": x["answer"], "label": x["label"], "plain": x["plain"]} + for x in r["limbs"]]}) + continue + if r["status"] != "date-open": + continue + out.append({ + "fact": "offence-date", "parameter": "offence_date", "code": r["code"], + "question": (f"When was the offence committed? '{r['code']}' names a " + "different offence depending on the date."), + "options": [{"value": b["period"], + "label": (f"{b['period']}: " + (f"'{b['catchline']}' " if b["catchline"] else "") + + ("-> " + " or ".join(f"'{p}'" for p in b["limbs"]) + + " (the specification decides)" if b.get("limbs") + else f"-> '{b['plain']}'" if b["plain"] + else "-> no curated pairing"))} + for b in r["readings"]]}) + return out + + +def has_military_cue(text): + """True when the words say UCMJ, court-martial or military -- the cue a + bare 'Article N' needs before it is read as a UCMJ article.""" + return bool(_MILITARY_CUE.search(_fold(text or ""))) + + +def is_single_term(offense, pairings, state=None): + """True when the whole string is itself one pairing term ('assault and + battery', 'breaking and entering'), so a separator inside it is not a + second offence.""" + text, _ = expand(offense, state) + for p in pairings: + for term in _terms(p, state): + if _fold(term).strip() == text: + return True + return False + + +def _phrases_in(text, pairings, state): + padded = f" {text} " + return {t for p in pairings for t in (_fold(x).strip() for x in _terms(p, state)) + if t and f" {t} " in padded} + + +# 'aiding and abetting' (and its forms) names ONE liability (roadmap T2.4, +# track A step A10): its 'and' is never an offence separator, whether or not a +# curated term carries the phrase (the approved baseline has none). +_AID_ABET = re.compile(r"\b(aid(?:ing|ed|s)?|aider)\s+(?:and|&)\s+(abet(?:ting|ted|s)?|abettor)\b", + re.I) +# A count suffix after a comma ('theft, 2 counts', 'DUI, x2', 'assault, 3 cts') is the +# same offence's count, not a second offence; a comma-space inside a digit group +# ('5, 000 grams') is the amount's (the audit follow-up of 2026-09-21). +# ('theft, 2 counts of burglary' names a second offence: a count followed by 'of' is +# not a suffix, and a multiplier beside an amount is not a count) +_COUNT_SUFFIX = re.compile( + r",\s*(x\s*\d+(?!\s*(?:[.,/]\d+)?\s*" + _UNIT_WORDS + r")|\d+\s*x(?!\s*\d)|" + r"\d+\s*(?:counts?|cts?\.?)|counts?\s*#?\s*\d+)\b(?!\s+of\b)", re.I) +# a multiplier before an amount ('2 x 2 kg', '3 x 50 lb bags'): the record's total is +# not the amount written, so quantity_kg names none +_MULTIPLIED_AMOUNT = re.compile(r"\d+\s*[x\u00d7]\s*\d+(?:[.,/]\d+)?\s*" + _UNIT_WORDS, re.I) +_DIGIT_GROUP_COMMA = re.compile(r"(\d),\s+(\d{3})\b") + + +def split_offences(offense, pairings=None, state=None): + """The separate offences in one record string, or [offense] when it is + one. '&' reads as 'and' first (roadmap T1.3). With the pairings, a + separator that sits inside a pairing term is not a split: the two sides + are re-joined when the joined text names a term neither side names alone + ('breaking & entering', 'B&E of a dwelling', 'assault and battery'), and + a string that is itself one term is never split. The 'and' of 'aiding and + abetting' is never a separator.""" + guarded = _AID_ABET.sub(lambda m: f"{m.group(1)}_and_{m.group(2)}", offense) + guarded = _DIGIT_GROUP_COMMA.sub(r"\1,\2", _COUNT_SUFFIX.sub(r" \1", guarded)) + pieces = _SPLIT_CAP.split(guarded) + parts, seps = [pieces[0]], pieces[1::2] + parts += pieces[2::2] + if pairings is not None: + if is_single_term(offense, pairings, state): + return [offense] + merged = [parts[0]] + for sep, nxt in zip(seps, parts[1:]): + joined = merged[-1] + sep + nxt + both = _phrases_in(expand(joined, state)[0], pairings, state) + alone = (_phrases_in(expand(merged[-1], state)[0], pairings, state) + | _phrases_in(expand(nxt, state)[0], pairings, state)) if both else set() + if both - alone: + merged[-1] = joined + else: + merged.append(nxt) + parts = merged + parts = [p.strip().replace("_and_", " and ") for p in parts if p and p.strip()] + return parts if len(parts) > 1 else [offense] + + +def _singular(tok): + """A crude singular for matching (track A step A9, gold misses 'BAD + CHECK' and 'TERRORISTIC THREAT'): applied to the query and to every + pairing term alike, so it only lets a singular reach a plural term (and + back). '-ss', '-us', '-is' and '-ous' endings and short tokens stay + whole ('cannabis', 'status', 'dangerous').""" + if len(tok) > 4 and tok.endswith("ies"): + return tok[:-3] + "y" + if len(tok) > 3 and tok.endswith("s") and not tok.endswith(("ss", "us", "is", "ous")): + return tok[:-1] + return tok + + +def content_tokens(text): + return {s for s in (_singular(t) for t in re.findall(r"[a-z0-9]+", text.lower()) + if t not in STOP) if s not in STOP} + + +@functools.lru_cache(maxsize=16384) +def _term_reading(term): + """(folded term, the term read as the record is read, its content tokens) + for one curated pairing term. Pure, and memoised (audit follow-up + 2026-09-21, optimization 1): _best_match read all 779 pairing terms -- + folding, expanding shorthand and tokenising each -- on every match, about + 60 % of a warm screening call. The tokens are a frozenset: the cached + value is shared.""" + tl = _fold(term).strip() + return tl, _expand_abbreviations(_expand_phrases(tl)).strip(), frozenset(content_tokens(tl)) + + +def _terms(p, state=None): + """A pairing's matching terms: us_terms plus any state_terms for the + record's state (state-qualified labels such as New York's 'assault in + the third degree', which is the misdemeanour there and a felony in + Washington).""" + terms = list(p.get("us_terms") or []) + if state: + st = state.strip().lower() + for key, extra in (p.get("state_terms") or {}).items(): + if key.lower() == st: + terms += list(extra) + return terms + + +# A term found in the query as a whole phrase is a stronger signal than one +# whose content words merely cover it: 'assault in the third degree' names +# New York's simple assault outright, while 'felony assault' reaches the +# aggravated pairing only because both reduce to the token {assault}. Token +# matches are therefore capped below a phrase match. +TOKEN_CAP = 0.95 + + +def _best_match(text, q, p, state=None): + """(score, kind, length) for the pairing's best-matching term, where kind + is 'phrase', 'tokens' or None.""" + best, kind, length = 0.0, None, 0 + military = None + for term in _terms(p, state): + tl, tx, tt = _term_reading(term) + if not tl: + continue + if _UCMJ_ARTICLE.match(tl): + # a bare UCMJ article number names nothing without a military cue + if military is None: + military = bool(_MILITARY_CUE.search(text)) + if not military: + continue + # the term read as the record is read (track A step A10): the record's + # 'attempt' is expanded to 'attempted', so the curated term 'attempt to + # evade or defeat tax' could never match as a phrase (tx, _term_reading) + if f" {tl} " in f" {text} " or (tx != tl and f" {tx} " in f" {text} "): + # The first phrase hit REPLACES any token-match length: a token + # match's term length is not a phrase length, and carrying it + # over made two identical phrase matches look unequal. + if kind != "phrase": + best, kind, length = 1.0, "phrase", len(tl) + elif len(tl) > length: + length = len(tl) + continue + inter = q & tt + if not tt or not inter: + continue + cov_term = len(inter) / len(tt) + cov_query = len(inter) / max(len(q), 1) + if cov_term >= 0.6 and cov_query >= 0.5: + score = min(round(0.5 * cov_term + 0.5 * cov_query, 3), TOKEN_CAP) + if kind == "phrase": + continue + if score > best: + best, kind, length = score, "tokens", len(tl) + elif score == best and len(tl) > length: + length = len(tl) + return best, kind, length + + +def match_pairings(offense, pairings, top=2, state=None): + """[(score, pairing)] for the pairings the query confidently names. + + A term matches when it appears in the query as a phrase (score 1.0) or + when the query's content words cover at least 60% of the term's AND the + term covers at least half of the query's (so 'driving' alone cannot + reach DUI, nor 'possession' alone reach cannabis) -- the latter capped + below a phrase match. A pairing's negative_terms veto it outright. + Equal scores are ordered by the longest matching term, then by id; a tie + that survives that is reported by is_tie, never resolved silently.""" + text, _ = expand(offense or "", state) + q = content_tokens(text) + scored = [] + for p in pairings: + if any(_fold(n).strip() in text for n in p.get("negative_terms") or []): + continue + best, kind, length = _best_match(text, q, p, state) + if best >= MIN_SCORE: + scored.append((best, length, p)) + scored.sort(key=lambda t: (-t[0], -t[1], t[2]["id"])) + return [(sc, p) for sc, _, p in scored[:top]] + + +def is_tie(scored, offense=None, state=None): + """True when the two best pairings matched equally well and the record's + own words cannot say which -- the caller must not pick one silently. + + Two phrase matches tie only at the same length ('sexual battery' beats + 'battery'; 'larceny' in both theft pairings is a real tie). Two token + matches always tie at the same score: term length means nothing there -- + 'misdemeanor assault' and 'felony assault' both reduce to {assault}, so + the longer string is not the better answer.""" + if len(scored) < 2: + return False + (s1, p1), (s2, p2) = scored[0], scored[1] + if s1 != s2: + return False + if offense is None: + return True + text, _ = expand(offense, state) + q = content_tokens(text) + _, k1, l1 = _best_match(text, q, p1, state) + _, k2, l2 = _best_match(text, q, p2, state) + if k1 != k2: + return False + return l1 == l2 if k1 == "phrase" else True + + +# The fact that splits a tie (roadmap T1.4). This is code, not curated data: +# each entry names the one fact that decides between two pairings the record's +# words fit equally, the tool parameter that carries it (None when no +# parameter does yet), and which member each answer selects -- read from the +# statute, whose words tests/test_screening_chain.py pins in the corpus +# (C-46 s. 334(a)/(b): 'more than $5,000' / 'not more than $5,000'; s. 320.17: +# 'operates a motor vehicle or vessel while being pursued by a peace officer'). +# A tie is recognised by its curated tie_group label OR by its member ids: +# the approved baseline (d784d67) carries both theft pairings without a +# tie_group, and a composed release must still resolve that tie on the value. +# A tie this map does not name is reported unresolved, never chosen. +TIE_FACTS = { + "theft-value": { + "members": ("theft-over", "theft-under"), + "fact": "value_over_5000", + "parameter": "value_over_5000", + "question": ("was the value of what was stolen more than CAD 5,000? " + "(Criminal Code s. 334(a): more than $5,000, 10 years; " + "s. 334(b): not more than $5,000, 2 years)"), + "options": ((True, "value more than CAD 5,000", "theft-over"), + (False, "value not more than CAD 5,000", "theft-under")), + "caveat": ("s. 334(a) also covers a testamentary instrument whatever " + "its value, and theft of a motor vehicle is s. 333.1 (listed " + "on theft-over): confirm what was stolen, not only its value."), + }, + "flight-mode": { + "members": ("evading-police", "resisting-arrest"), + "fact": "operated_motor_vehicle_or_vessel", + "parameter": None, + "question": ("was the person operating a motor vehicle or vessel while " + "being pursued by a peace officer? (flight: Criminal Code " + "s. 320.17; otherwise resisting or obstructing a peace " + "officer, s. 129) -- no parameter carries this fact: re-run " + "with the charge stated more specifically"), + "options": ((True, "operating a motor vehicle or vessel while pursued", + "evading-police"), + (False, "not operating a motor vehicle or vessel", + "resisting-arrest")), + "caveat": None, + }, +} + +_GENERIC_TIE_QUESTION = ( + "which pairing's elements the record proves: give a more specific charge, " + "the statute, or the facts (Hill's evidence branch), and re-run") + + +def tie_spec(p1, p2): + """(name, spec) for two tied pairings: the TIE_FACTS entry named by their + shared tie_group, or the one whose members are exactly these two ids. + (group or None, None) when the map names neither -- a tie group the + approved baseline lacks, or one not in the map, is tolerated, not an + error.""" + ids = {p1["id"], p2["id"]} + g1, g2 = p1.get("tie_group"), p2.get("tie_group") + group = g1 if g1 and g1 == g2 else None + for name, spec in TIE_FACTS.items(): + if (group == name) or set(spec["members"]) == ids: + return name, spec + return group, None + + +def resolve_tie(scored, facts=None, offense=None, state=None, term_notes=None): + """One tie rule for every step-2 caller (roadmap T1.4): the matched + pairing, the tied one when the record's words cannot choose, and what + would settle it. + + Returns {"tied", "pairing", "other", "group", "resolution", "needs"}: + tied True when two pairings still fit equally -- the caller must + report branch 'tie' and no band, never the first pairing's; + pairing the chosen pairing (the first one when tied; None when + nothing matched); + other the tied pairing when tied, else None; + resolution when a supplied fact settled the tie: {parameter, value, + label, chosen, other, caveat}; + needs [{fact, parameter, question, options}] naming what decides + an unresolved tie (a curated term note's element split + when one covers both pairings). + facts: the tool parameters the record supplied, e.g. + {"value_over_5000": False}.""" + out = {"tied": False, "pairing": scored[0][1] if scored else None, + "other": None, "group": None, "resolution": None, "needs": []} + if not scored or not is_tie(scored, offense, state): + return out + p1, p2 = scored[0][1], scored[1][1] + group, spec = tie_spec(p1, p2) + out["group"] = group + facts = facts or {} + if spec and spec.get("parameter") and facts.get(spec["parameter"]) is not None: + value = bool(facts[spec["parameter"]]) + for opt_value, label, pid in spec["options"]: + chosen = next((p for p in (p1, p2) if p["id"] == pid), None) + if opt_value == value and chosen is not None: + other = p2 if chosen is p1 else p1 + out.update(pairing=chosen, resolution={ + "parameter": spec["parameter"], "value": value, "label": label, + "chosen": chosen["id"], "other": other["id"], + "caveat": spec.get("caveat")}) + return out + out.update(tied=True, other=p2) + ids = {p1["id"], p2["id"]} + if spec: + out["needs"] = [{"fact": spec["fact"], "parameter": spec["parameter"], + "question": spec["question"], + "options": [{"value": v, "label": lab, "pairing": pid} + for v, lab, pid in spec["options"] + if pid in ids]}] + return out + note = next((n for n in term_notes or [] + if ids <= set(n.get("pairings") or [])), None) + question = (f"the element in the curated note {note.get('id')} (printed " + "below as ELEMENT SPLIT) decides it" if note else _GENERIC_TIE_QUESTION) + out["needs"] = [{"fact": "elements", "parameter": None, "question": question, + "options": [{"value": None, "label": p["id"], "pairing": p["id"]} + for p in (p1, p2)]}] + return out + + +def occurrence_range(items): + """How many separate occurrences a set of convictions can be, for the + two-offence limbs of IRPA s. 36(2)(a)/(b) ('two offences not arising out + of a single occurrence' -- critic, 2026-09-12). + + items: [(label, occurrence_group or None, offence_date or None)]. + Two items are certainly ONE occurrence when both carry the same + occurrence_group; certainly SEPARATE when both carry different groups, or + when neither group settles it and their offence dates differ. Equal or + missing offence dates leave it open. Returns (least, most, open_pairs): + the fewest and most occurrences the items can be, and the label pairs + left open. The count of two is settled only when least >= 2 (separate) + or most <= 1 (one occurrence). + + 'Possibly the same occurrence' is not transitive -- an undated item may be + one occurrence with a 2010 item or with a 2012 item, never with both -- so + the fewest occurrences is the smallest number of classes the certainly-same + blocks can be put in with every pair in a class possibly the same (an exact + search over the blocks; the code audit of 2026-09-21). Before it, the blocks + were merged transitively and A(2010), B(undated), C(2012) counted as + possibly ONE occurrence, so the tools asked for occurrence_group where the + two dated convictions already settled 'not arising out of a single + occurrence'. Past _OCCURRENCE_EXACT_MAX blocks the transitive count (never + too high) is kept.""" + n = len(items) + + def blocks_of(same): + parent = list(range(n)) + + def find(i): + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + for i in range(n): + for j in range(i + 1, n): + if same(items[i], items[j]): + parent[find(i)] = find(j) + groups = {} + for i in range(n): + groups.setdefault(find(i), []).append(i) + return list(groups.values()) + + def certainly_same(a, b): + return a[1] is not None and b[1] is not None and a[1] == b[1] + + def possibly_same(a, b): + if a[1] is not None and b[1] is not None: + return a[1] == b[1] + return a[2] is None or b[2] is None or a[2] == b[2] + + def least_classes(blocks): + # the minimum clique cover of the certainly-same blocks under possibly_same + compatible = [[all(possibly_same(items[x], items[y]) for x in a for y in b) + for b in blocks] for a in blocks] + best = [len(blocks)] + + def place(k, classes): + if len(classes) >= best[0]: + return + if k == len(blocks): + best[0] = len(classes) + return + for cls in classes: + if all(compatible[k][m] for m in cls): + cls.append(k) + place(k + 1, classes) + cls.pop() + classes.append([k]) + place(k + 1, classes) + classes.pop() + place(0, []) + return best[0] + + open_pairs = [(items[i][0], items[j][0]) for i in range(n) for j in range(i + 1, n) + if possibly_same(items[i], items[j]) + and not certainly_same(items[i], items[j])] + certain = blocks_of(certainly_same) + if len(certain) <= _OCCURRENCE_EXACT_MAX: + least = least_classes(certain) + else: + # the audit follow-up of 2026-09-21: past the bound, the transitive count is + # raised to two cheap valid lower bounds -- the distinct groups, and the + # distinct offence dates of the ungrouped items -- since either set is pairwise + # certainly separate (13 summary convictions with 6 undated counted 'least 1' + # and the tools left the IRPR s. 18(2) count unsettled) + groups = {it[1] for it in items if it[1] is not None} + dates = {it[2] for it in items if it[1] is None and it[2] is not None} + least = max(len(blocks_of(possibly_same)), len(groups), len(dates)) + return least, len(certain), open_pairs + + +_OCCURRENCE_EXACT_MAX = 12 + + +def suggest_by_meaning(offense, index, threshold=0.83): + """The closest pairing chunk by embedding, as a labelled suggestion: + (pairing id, cosine) or None. Needs the loaded index (its embedder and + the commentary chunk vectors); never asserts a match.""" + if index is None or not getattr(index, "semantic", False): + return None + np = index._np + rows = [r for r, o in enumerate(index._row_chunk.tolist()) + if index.chunks[o]["id"].startswith("commentary-equiv-")] + if not rows: + return None + qv = index.embedder.encode_query(offense) + sims = index.vectors[rows] @ qv + best = int(np.argmax(sims)) + if float(sims[best]) < threshold: + return None + chunk = index.chunks[int(index._row_chunk[rows[best]])] + return chunk["id"][len("commentary-equiv-"):], round(float(sims[best]), 3) + + +# ---- limbs (roadmap T1.13 / T3.1) ----------------------------------------- +# A pairing may name several Canadian provisions whose maxima have different +# histories (child pornography: possession 5 years until 2015-07-16, making +# 10; cannabis: trafficking in 3 kg or less 5 years less a day before +# 2018-10-17, more than 3 kg life). The curated, optional 'limbs' list names +# them: {"key", "label", "terms": [phrases that select it], "quantity": +# "3kg-or-less" | "over-3kg" (optional), "max_years" (optional: the limb's +# own current maximum when it differs from the pairing's)}. A history row +# with a "limb" key applies only to that limb; a row without one applies to +# the whole pairing -- so the approved baseline, which has no limbs, keeps its +# meaning. Which limb a record is comes from its words (and, for the CDSA +# quantity line, an amount written in it); when the words do not say and the +# limbs give different branches, both readings are reported, never one. + +# The quantity line of former CDSA s. 5(4) / 5(3)(a.1): the Schedule VII +# amount for cannabis (marihuana) and cannabis resin, 3 kg -- read from the +# point-in-time schedule pages (data/curated/drafts/pit-report-2026-09-12.json +# schedule_checks; pinned by tests/test_tran_limbs.py). +QUANTITY_LINE_KG = 3.0 +QUANTITY_LIMBS = {"3kg-or-less": lambda kg: kg <= QUANTITY_LINE_KG, + "over-3kg": lambda kg: kg > QUANTITY_LINE_KG} +# Units written on US records, in kilograms (mechanical conversions). +_KG_PER_UNIT = {"kg": 1.0, "kgs": 1.0, "kilo": 1.0, "kilos": 1.0, + "kilogram": 1.0, "kilograms": 1.0, + "g": 0.001, "gr": 0.001, "gm": 0.001, "gms": 0.001, + "gram": 0.001, "grams": 0.001, "mg": 0.000001, + "lb": 0.45359237, "lbs": 0.45359237, "pound": 0.45359237, + "pounds": 0.45359237, + "oz": 0.028349523125, "ozs": 0.028349523125, + "ounce": 0.028349523125, "ounces": 0.028349523125} +# a digit group separator: the US comma, the French/SI space (also the no-break +# and narrow no-break spaces a copied record carries) or a comma with a space +_GROUP_SEP = r"(?:,\s?|[ \u00a0\u202f])" +_AMOUNT = re.compile(r"(?= float(b): + return None # not a proper fraction: no mixed-number reading + return (float(whole) if whole.strip() else 0.0) + float(a) / float(b) + if _GROUPED_AMOUNT.fullmatch(num): + return float(_GROUP_SEP_RE.sub("", num).replace(",", ".")) + return float(num.replace(",", ".")) + + +# The verification fixes of 2026-09-22 (F1: LEGAL-1/2/3, HELD): an amount settles the +# quantity only when the words state ONE exact amount. A bound ('over 5 lbs', 'at least 2 +# kg', 'less than 50 lbs', 'up to 5 kg', '5 lbs+', '>5 lbs'), an approximation ('approx. +# 3 kg', 'about', 'around', 'roughly', 'some', 'estimated'), a range ('5-10 lbs', '2 to 5 +# kg', 'between'), a multiplier ('2 kilos x 2', '2 x 28 g'), a per-unit count ('200 bags of +# 20 g each', '3 bricks of 1 kg') or a gross weight ('3.5 kg gross', 'including +# packaging') anywhere in the offence text leaves the quantity open -- both limbs are +# reported (NOT SETTLED, 'how much was involved?'), the strict direction. Before, each was +# read as the exact amount written: 'over 5 lbs' settled the 3-kg-or-less limb (a +# permanent resident NOT REACHED) and 'less than 50 lbs' the over-3-kg one. Since +# verification round G1 this list is only a pre-check that opens: what settles a limb +# is the ALLOWLIST (settled_quantity, below quantity_kg), which the list could not be -- +# 189 of 678 phrasings the verification pass tried still settled past it. +_AMOUNT_QUALIFIER = re.compile( + r"\b(?:over|under|above|below|more|less|fewer|greater|excess|exceed(?:s|ed|ing)?|" + r"least|most|min|minimum|max|maximum|up\s+to|approx|approximate|approximately|about|" + r"around|roughly|some|estimate|estimated|circa|nearly|almost|between|gross|packag\w*|" + r"including|incl|each|per|plus|bags?|baggies|bales?|blocks?|bricks?|bundles?|" + r"parcels?|plants?|pills?|tablets?|units?|doses?|jars?|boxes|containers?)\b" + r"|[+<>~±≈≤≥]" + r"|\d\s*[-–—]\s*\d(?!\d*/\d)" + r"|\d\s*(?:[a-z]+\s+)?(?:to|through|thru)\s+\d" + r"|(? 0.907. A comma + followed by three digits is a thousands separator, as US records write it + ('5,000 grams' is 5 kg), and so is a space between digit groups, as French + and SI records write it ('5 000 grams' is 5 kg); a comma followed by one or + two digits is a decimal ('2,5 kg' is 2.5 kg); '1/2 oz' and '1 1/2 lbs' are + read as fractions. Before the code audit of 2026-09-21 every comma was a + decimal point and '5,000 grams' read 0.005 kg, flipping the cannabis + quantity limb; its follow-up read '5 000 grams' as 0 kg (the space-grouped + amount had left the matching text but not reached the reader), so a + settled 3-kg-or-less answer came from no amount at all. + + Verification round G1 (2026-09-22, Q9): a number whose dot is followed by + exactly three digits ('4.000 g', '2.500 kg') reads two ways -- a decimal + point (4 g) or a dot thousands separator, as continental records write it + (4 kg) -- and names no amount; so does an amount written right after a + comma or a dot ('3.000,5 g', where only '5 g' matched and read 0.005 kg). + This reader never settles a limb on its own: select_limbs settles only + through settled_quantity, the allowlist.""" + text = offense or "" + if _MULTIPLIED_AMOUNT.search(text) or _AMOUNT_QUALIFIER.search(text): + return None + found = set() + for m in _AMOUNT.finditer(text): + num, unit = m.group(1), m.group(2) + if _DOT_GROUPED.fullmatch(num) or (m.start() and text[m.start() - 1] in ".,"): + return None # Q9: two readings, or the tail of a longer number + n = _amount_number(num) + if n is None or not math.isfinite(n): + continue # a zero denominator or a 400-digit run: no amount + kg = round(n * _KG_PER_UNIT[unit.lower()], 6) + if kg > 0: + found.add(kg) + return found.pop() if len(found) == 1 else None + + +# ---- the quantity ALLOWLIST (verification round G1, 2026-09-22) ------------ +# F1 opened the quantity when the text carried a word from a list +# (_AMOUNT_QUALIFIER). A list of doubtful words is never complete: the next +# pass found 189 of 678 phrasings it missed ('est. 2.9 kg', 'GT 2 KG', 'five +# packs of 1 kg', '2 kg sample of larger load', 'four kilograms 200 grams'), +# each settling the 3-kg-or-less limb. The rule is now the reverse: an amount +# settles a quantity limb only when the record's words are sure of it -- +# * the offence text holds exactly ONE amount (_AMOUNT), and quantity_kg +# reads it (its word list stays, as a pre-check that only opens); +# * outside that amount the raw text has no digit and no symbol: letters, +# spaces and ( ) . , ; : ' " / only (a second number, a count, a date, a +# citation, '~', '+', '?', '<', '-', '@' all leave it open); +# * no character of the raw text is one the folder erases (GQ-4, below: +# _fold keeps a-z, digits and ( ) . / & ' only, so a word in another +# script -- '2 kg 以上' = 'or more', '約 2 kg' = 'approx.' -- would +# otherwise stand beside the amount unread); +# * those words, read as the matcher reads them (expand()), are nothing but +# the QUANTITY limbs' own offence words (quantity_limb_terms; GQ-2: a word +# only a limb WITHOUT a quantity writes -- 'manufacture', 'cultivation', +# 'growing' -- names a different Canadian provision, so it opens), the two +# substances former Schedules VII and VIII name +# (SCHEDULE_VII_WORDS) and QUANTITY_NEUTRAL_WORDS, once the ONE inchoate or party cue and +# the military jurisdiction the matcher itself reads are set aside +# (_offence_tokens: 'attempted', 'conspiracy to', 'UCMJ' -- liability and +# jurisdiction, not amount). A function word the terms use ('to', +# 'with', 'for') counts only inside a run of two or more words the terms +# write in that order ('with intent to distribute'), never on its own; +# * those words name ONE offence clause (one_offence_clause, GQ-3: an +# allowlist of words alone is a bag of words -- a second count on a new +# line, a repeated term head ('1 lb a sale') or a second substance with no +# amount of its own all left the amount 'settled'); +# * the text does not end, after the amount, on a function word ('2 kg to', +# '2 kg of'): a record cut off mid-phrase; +# * the facts and the disposition name no other amount and qualify none +# (quantity_context_open, GQ-5: 'quantity: over 3 kg' in the facts is the +# officer's own answer and must never leave a 3-kg-or-less settle standing). +# Everything else leaves the quantity open -- NOT SETTLED, both readings +# shown. Nothing is recognised one phrasing at a time, so a phrasing no one +# listed cannot settle: it opens. +QUANTITY_NEUTRAL_WORDS = frozenset({"a", "an", "the", "of", "in"}) +# the characters that may stand outside the one amount +_REST_PUNCT = frozenset("().,;:'\"/") +# unit words: a curated term naming one never lends its words to the allowlist +_UNIT_TOKENS = frozenset(_KG_PER_UNIT) | {"ml"} +# the characters _fold keeps: everything else it replaces with a space +_FOLD_KEEP = frozenset("abcdefghijklmnopqrstuvwxyz0123456789()./&' ") + + +def _plain_chars(text, digits=False): + """True when the text is letters, spaces and _REST_PUNCT only (and digits + when digits=True).""" + return all(ch.isalpha() or ch.isspace() or ch in _REST_PUNCT + or (digits and ch.isdigit()) for ch in text) + + +def fold_erases_word(text): + """True when the text carries a character _fold would erase (GQ-4, + verification round H1): _plain_chars accepts every Unicode letter, and + _fold keeps only a-z, digits and ( ) . / & ' -- so a qualifier in another + script ('2 kg 以上' = 'or more', '約 2 kg' = 'approx.', '(приблизительно)') + never reached offence_words_only and the amount settled. A word the reader + cannot read must open the quantity, never vanish. Accents fold first, so + 'portación' and 'héroïne' are read, not erased; the comparison is by + character rather than by word count so a word fused to an erased run + ('marijuana約') opens too.""" + norm = unicodedata.normalize("NFKD", (text or "").lower()) + norm = "".join(ch for ch in norm if not unicodedata.combining(ch)) + return any(not (ch.isspace() or ch in _FOLD_KEEP or ch in _REST_PUNCT) for ch in norm) + + +def quantity_terms(pairing): + """The pairing's own offence words the allowlist is derived from: its + us_terms, every state's state_terms and its limbs' terms -- except a term + that itself names a number, a unit or a qualifier (_AMOUNT_QUALIFIER), + whose words could otherwise pass a bound as offence words.""" + terms = list(pairing.get("us_terms") or []) + for extra in (pairing.get("state_terms") or {}).values(): + terms += list(extra or []) + for limb in limb_entries(pairing): + terms += list(limb.get("terms") or []) + out = [] + for t in dict.fromkeys(terms): + toks = re.findall(r"[a-z0-9]+", _term_reading(t)[1]) + if (toks and not any(ch.isdigit() for ch in t) + and not _UNIT_TOKENS & set(toks) and not _AMOUNT_QUALIFIER.search(t)): + out.append(tuple(toks)) + return tuple(out) + + +def quantity_limb_terms(pairing): + """The words the ALLOWLIST is derived from when an amount is read: the + terms of the pairing's QUANTITY limbs only. + + GQ-2 (verification round H1): quantity_terms collects the pairing's + us_terms, its state_terms and EVERY limb's terms, so a word only a limb + WITHOUT a quantity writes lent itself to the allowlist -- 'manufacture + delivery of marijuana 2 kg', 'growing sale of marijuana 2 kg' and + 'cultivation sale of marijuana 2 kg' each settled the 3-kg-or-less + trafficking limb and the production reading disappeared. Production of + cannabis was its own provision (former CDSA s. 7(2)(b): 7 years to + 2012-11-05, then 14), so a production word beside the amount does not show + that the amount is the amount of a trafficking offence: the quantity + opens, and both readings are reported. A pairing with no quantity limb + (cannabis-possession, which possession_limit_open reads) keeps its own + words.""" + qlimbs = [l for l in limb_entries(pairing) if l.get("quantity") in QUANTITY_LIMBS] + return quantity_terms({"limbs": qlimbs}) if qlimbs else quantity_terms(pairing) + + +@functools.lru_cache(maxsize=1024) +def _term_pieces(seqs): + """(content words, runs): the terms' words that are not function words + (STOP), and every run of two or more words a term writes in order.""" + content = frozenset(t for s in seqs for t in s if t not in STOP) + runs = frozenset(s[i:j] for s in seqs for i in range(len(s)) + for j in range(i + 2, len(s) + 1)) + return content, runs + + +def offence_words_only(tokens, seqs, extra=frozenset()): + """True when every token is a content word of the terms `seqs`, a neutral + word, one of `extra`, or lies in a run of two or more tokens that a term + writes in the same order.""" + content, runs = _term_pieces(tuple(seqs)) + covered = [False] * len(tokens) + for i in range(len(tokens)): + for j in range(i + 2, len(tokens) + 1): + if tuple(tokens[i:j]) in runs: + for k in range(i, j): + covered[k] = True + return all(c or t in content or t in QUANTITY_NEUTRAL_WORDS or t in extra + for t, c in zip(tokens, covered)) + + +def _cut_off(after): + """True when the words after the amount end on a function word ('2 kg + to', '2 kg of', '2 kg in'): the record was cut off mid-phrase. A count + ('2 counts') is not read as words here.""" + words = re.findall(r"[a-z0-9]+", _COUNT_WORDS.sub(" ", _fold(after))) + return bool(words) and (words[-1] in STOP or words[-1] in QUANTITY_NEUTRAL_WORDS) + + +# an offence clause ends at a newline, a full stop or a colon ('sale of +# marijuana 2 kg\nsale of marijuana' is two counts, not one offence). A dot +# between digits is a decimal, never a boundary. +_CLAUSE_BOUNDARY = re.compile(r"[\n\r:]|(? 'sale', 'marijuana + distribution' -> 'distribution'). Two of one head name two counts.""" + heads = set() + for s in seqs: + for t in s: + if t not in STOP and t not in _SUBSTANCE_WORDS: + heads.add(t) + break + return frozenset(heads) + + +def _substance_groups(folded): + """How many separate substance word groups the folded text names. + Adjacent substance words are ONE group ('cannabis resin', 'marijuana + (cannabis)' -- punctuation does not separate them); any other word ends + the group, so 'marijuana with hashish' is two.""" + groups, run = 0, False + for tok in re.findall(r"[a-z0-9]+|[^\sa-z0-9]", folded): + if tok in _SUBSTANCE_WORDS: + groups += 0 if run else 1 + run = True + elif tok[0].isalnum(): + run = False + return groups + + +def one_offence_clause(rest, seqs, extra=frozenset()): + """True when the words around the amount name ONE offence clause, so the + amount is the amount of the offence being screened. + + GQ-3 (verification round H1): offence_words_only asks only whether every + token is an allowed word, so a bag of allowed words settled -- a second + count pasted on a new line or after a full stop ('sale of marijuana 2 + kg\\nsale of marijuana'), a per-transaction tail ('1 lb a sale', '1 + lb/sale') and a second substance with no amount of its own ('sale of 2 kg + of marijuana with hashish') each left ONE weight standing for the whole + record. Three structural tests, all conservative: + * only one clause (newline, '.' or ':') names an offence word; + * no term's head word occurs twice in the matcher's reading; + * the record names at most one substance group (the second has no + amount of its own). + Any of them failing opens the quantity -- NOT SETTLED, both limbs.""" + content, _ = _term_pieces(tuple(seqs)) + offence_words = content | frozenset(extra) | _SUBSTANCE_WORDS + named = [c for c in _CLAUSE_BOUNDARY.split(rest or "") + if offence_words & set(re.findall(r"[a-z0-9]+", _fold(c)))] + if len(named) > 1: + return False + tokens = _offence_tokens(expand(rest)[0]) + if any(tokens.count(h) > 1 for h in _term_heads(tuple(seqs))): + return False + return _substance_groups(_fold(rest or "")) <= 1 + + +# ---- Q12: the tokens the allowlist REMOVES before it reads the words ------- +# G1's allowlist asks that outside the one amount the raw text hold no digit +# (_plain_chars), so an EXACT amount beside a record-keeping token stayed NOT +# SETTLED although the offence's own words settle it: 'SALE OF CANNABIS +# 893.13(1)(a)(2) 2 KG', 'sale of marijuana 2 kg 2015-06-01', 'sale of +# marijuana 2 kg on 06/01/2015', 'sale of marijuana 2 kg Case 2015CF001234', +# 'sale of marijuana 2 kg Count 1' and 'sale of marijuana 1-1/2 lbs' -- about +# 56 of the quantity lens's 678 realistic phrasings. +# +# The fix stays inside the allowlist: a token this grammar RECOGNISES is +# removed before the allowlist reads the words, so what is left is the +# offence's own words plus one amount. Nothing new is recognised as an amount +# or as an offence word, and a token the grammar cannot classify still opens +# the quantity. Each removable token carries no information about HOW MUCH: +# * a statute citation -- a designator on the explicit allowlist +# _Q_CODE_DESIGNATORS that the alias table also carries +# (_citation_designators, so 'HS', 'HSC', 'RCW', 'MCL', 'USC', 'PC', 'VC' +# but NOT 'ORS', 'KRS', 'ARS', 'G.S.' or 'R.C.', which the table lacks, and +# NOT an offence name or another legal system's code such as 'estafa', +# 'IPC', 'NDPS Act' or the bare word 'article', which the allowlist +# excludes -- all of them keep opening the +# quantity), with an optional title number, a section number and subsection +# letters; a bare section number with subsections ('893.13(1)(a)(2)'); or a +# run of two or more bare subsections ('(b)(1)(D)'). One group alone is +# never a citation, so '(THC)', '(F3)' and '(10 packs)' still open; +# * an ISO date, a slash date, with an optional label ('on', 'sentenced'); +# * a docket, case or cause number; +# * a count IDENTIFIER -- 'Count 1', 'Count #2', which says WHICH count this +# entry is. The CARDINAL form ('2 kg, 3 counts') is NOT removed: it says +# the offence happened more than once, so the weight written may be one +# count's or the whole record's, exactly the reading _AMOUNT_QUALIFIER +# opens for 'on three occasions'. K2 (2026-09-23, J2-L3): 'Count 2 of 3' +# states the same cardinal fact, so a count followed by 'of' is no +# identifier at all and nothing is removed -- the quantity opens exactly as +# '3 counts' does. +# A token is removed only where it overlaps NO _AMOUNT match, so a citation +# whose text an amount runs into ('sale of marijuana 1 G.S. 90.95', where +# _AMOUNT reads '1 G' as one gram) is left alone and keeps the quantity open; +# and, K2 (2026-09-23, J2-I6), only where it does not CUT INTO an +# _AMOUNT_QUALIFIER match of the words as written -- a citation's optional +# title number could otherwise swallow a multiplier's operand ('sale of +# marijuana 2 kg x 2 HS 11360' stripped to '... 2 kg x', which no longer +# carries the qualifier that must open the quantity). A qualifier match wholly +# INSIDE a removable token is part of that token, not a qualifier of the +# amount ('Docket 15-1234', '2015-06-01'), so it does not stop the removal. +_Q_ISO_DATE = r"\d{4}-\d{2}-\d{2}" +_Q_SLASH_DATE = r"\d{1,2}/\d{1,2}/(?:\d{4}|\d{2})" +_Q_DATE_LABEL = r"(?:on|in|dated?|filed|sentenced|committed|occurred|arrested)" +_Q_DATE = re.compile(rf"(?: ' -- was discarded while + the favourable settled answer stood: 'sale of marijuana 2 kg' with facts + 'quantity: over 3 kg' still answered the 3-kg-or-less limb, and the card + said 'The answer changes with: none found'. An amount there that is not + the offence text's (kg: the amount the offence text states, None when it + states none), or a qualifier beside an amount or a quantity word, opens + the quantity -- CanLex still settles it only from the OFFENCE TEXT.""" + text = context or "" + if not text.strip(): + return False + amounts = set() + for m in _AMOUNT.finditer(text): + n = _amount_number(m.group(1)) + if n is not None and math.isfinite(n): + amounts.add(round(n * _KG_PER_UNIT[m.group(2).lower()], 6)) + if any(a != kg for a in amounts): + return True + return bool(_AMOUNT_QUALIFIER.search(text) + and (amounts or _QUANTITY_WORDS.search(text))) + + +# the nouns a fact about the AMOUNT OF THE DRUG is written with: a qualifier +# beside one ('quantity: over', 'gross weight', 'never weighed') opens the +# quantity even where no amount is written. Deliberately narrow -- 'amount', +# 'value' and 'total' are the words of a sentence, a fine or restitution, and +# a qualifier beside those says nothing about the drug. +_QUANTITY_WORDS = re.compile(r"\b(?:quantit(?:y|ies)|weights?|weighed|weighing)\b", re.I) + + +def _offence_tokens(words): + """The tokens of the matcher's text with the ONE inchoate or party cue it + reads set aside ('attempted', 'conspiracy to': liability, _liability_cue) + and the military jurisdiction it reads ('UCMJ', 'court-martial': + _MILITARY_CUE -- the screening tools prefix 'UCMJ' to the offence of a + court-martial record). Neither is ever a word about the amount.""" + cue = _liability_cue(words) + if cue: + words = words[:cue[1].start()] + " " + words[cue[1].end():] + return re.findall(r"[a-z0-9]+", _MILITARY_CUE.sub(" ", words)) + + +# Q5 (verification round G1): former CDSA Schedule VII set an amount only for +# 'Cannabis resin 3 kg' and 'Cannabis (marihuana) 3 kg' (point-in-time +# 2002-12-31 and 2018-06-21, data/raw/pit/C-38.8/*/P1TT3xt3.html); Schedule II +# item 1 is 'Cannabis, its preparations and derivatives, including (1) +# Cannabis resin (2) Cannabis (marihuana) ... (7) Tetrahydrocannabinol'. A +# record naming THC or another preparation never settles the 3-kg-or-less +# limb; an exact amount over 3 kg still settles the over-3-kg limb (s. 5(3)(a), +# the same provision whichever the substance). 'hash', 'hashish' and 'resin' +# stay on the resin reading. +SCHEDULE_VII_FORMS = frozenset({ + "thc", "tetrahydrocannabinol", "oil", "oils", "concentrate", "concentrates", + "extract", "extracts", "wax", "shatter", "dab", "dabs", "vape", "vapes", + "cartridge", "cartridges", "cart", "carts", "edible", "edibles", "tincture", + "tinctures", "rosin", "budder", "distillate"}) +SCHEDULE_VII_FORM_NOTE = ( + "the record names {forms}, which may not be cannabis resin or cannabis (marihuana): " + "former CDSA Schedule VII listed only 'Cannabis resin 3 kg' and 'Cannabis (marihuana) " + "3 kg', and the five-years-less-a-day provision (s. 5(4), from 2012-11-06 " + "s. 5(3)(a.1)) reached a Schedule II substance only 'in an amount that does not exceed " + "the amount set out for that substance in Schedule VII' -- Schedule II item 1 is " + "'Cannabis, its preparations and derivatives', including '(7) Tetrahydrocannabinol'. " + "An amount of 3 kg or less does not settle the 3-kg-or-less limb for this record: get " + "the substance's form") +# the words of the two substances former CDSA Schedules VII and VIII name ('Cannabis +# resin', 'Cannabis (marihuana)'): the substance, never the amount, so they may stand +# beside it ('sale of cannabis resin 2 kg', 'possession of cannabis resin 0.5 g') +SCHEDULE_VII_WORDS = frozenset({"cannabis", "resin", "marihuana", "marijuana"}) +# Every word that names the substance or its form and nothing else: cannabis and +# its slang (_CANNABIS_SLANG), the Schedule VII and VIII substances, the forms +# Schedule VII set no amount for, and the street and brand names SUBSTANCES +# reads. A part of a split record whose content words are ONLY these names no +# offence of its own (GQ-1, plain_offence); two groups of them with one amount +# leave that amount unattached (GQ-3, _substance_groups). +_SUBSTANCE_WORDS = (SCHEDULE_VII_WORDS | SCHEDULE_VII_FORMS | _CANNABIS_CONCENTRATE + | frozenset(SUBSTANCES) + | frozenset({"weed", "pot", "thc", "hashish", "hash", "marihuana"})) +QUANTITY_HELD_NOTE = ( + "the amount is written beside words of the record string that are not an offence of " + "their own, or beside another amount: read with them it is not ONE exact amount, so " + "the quantity is not settled on this part") +QUANTITY_CONTEXT_NOTE = ( + "the facts or the disposition name an amount the offence text does not state, or " + "qualify one: read with them the amount written in the offence text is not shown to " + "be the exact amount of this offence, so the quantity is not settled. CanLex reads " + "the quantity from the OFFENCE TEXT only") + + +def schedule_vii_forms(offense): + """The words of the record naming a cannabis form Schedule VII sets no + amount for (SCHEDULE_VII_FORMS), in order, once each.""" + return [w for w in dict.fromkeys(re.findall(r"[a-z]+", _fold(offense or ""))) + if w in SCHEDULE_VII_FORMS] + + +# Q10/Q13 (verification round G1): the instruction for an open quantity said 'write the +# amount into the offence text (for example '2 kg') and re-run', and an officer holding +# 'over 5 lbs' re-ran with '5 lb.' and got the 3-kg-or-less answer by hand. +QUANTITY_EXACT_INSTRUCTION = ( + "write the EXACT amount the record states into the offence text, as the record " + "states it (for example '2 kg'), and re-run -- a bound ('over 5 lbs'), an estimate " + "('approx. 2 kg') or a part of the total cannot be written as an amount; with none, " + "the readings stand") + + +def _form_note(forms): + return SCHEDULE_VII_FORM_NOTE.format(forms=", ".join(f"'{w}'" for w in forms)) + + +def _no_exact_amount(forms=(), why=None): + lead = (QUANTITY_HELD_NOTE if why == "held" + else QUANTITY_CONTEXT_NOTE if why == "context" + else "the offence text states no amount CanLex can settle") + text = lead + " (" + QUANTITY_EXACT_INSTRUCTION + ")" + if forms: + text += "; and " + _form_note(forms) + return text + + +def quantity_open_why(forms=(), why=None): + """Why the quantity leaves the limb open, for the memo's LIMB NOT SELECTED + line (select_limbs 'forms' and 'quantity_why').""" + if why == "form" and forms: + return _form_note(forms) + return "the quantity decides between them and " + _no_exact_amount(forms, why) + + +def quantity_question(names, forms=(), why=None): + """The 'quantity' need's question: the provisions it decides between, and + what settles it (QUANTITY_EXACT_INSTRUCTION, or the substance's form).""" + if why == "form" and forms: + return ("was the substance cannabis resin or cannabis (marihuana), and how much? " + f"The form and the quantity decide the Canadian provision ({names}) -- " + + _form_note(forms)) + return (f"how much was involved? The quantity decides the Canadian provision ({names}), " + "and " + _no_exact_amount(forms, why)) + + +# Q4 (verification round G1): canlex_us_equivalency screens 'sale of marijuana +# 2 kg and over' as two parts ('sale of marijuana 2 kg'; 'over') and settled +# the first part's amount. While quantity_held() is in force, select_limbs +# settles no quantity: _equivalency_report holds it for a part whose other +# parts are not plain offences of their own (plain_offence). +_QUANTITY_HELD = contextvars.ContextVar("canlex_quantity_held", default=False) + + +@contextlib.contextmanager +def quantity_held(held=True): + token = _QUANTITY_HELD.set(bool(held)) + try: + yield + finally: + _QUANTITY_HELD.reset(token) + + +def quantity_is_held(): + return _QUANTITY_HELD.get() + + +def plain_offence(part, pairings, state=None): + """True when a part of a split record string is an offence of its own and + nothing else: it names no amount, no unit, no number or symbol outside a + statute code or a count the matcher reads, no character the folder erases, + its content words are not only the substance's names, and its words are + nothing but the matched pairing(s)' offence words and + QUANTITY_NEUTRAL_WORDS (the same grammar as settled_quantity). 'DUI' is; + 'over', 'approx.', '10 kg seized', 'at least' and 'hashish' are not. + + GQ-1 (verification round H1): a bare cannabis substance passed, because + expand() rewrites it to 'possession of ' (CANNABIS_BARE_NOTE) + and it then scored as cannabis possession with offence words only. 'sale of + marijuana 2 kg and hashish' therefore settled the 3-kg-or-less limb on the + first part and read the second as simple possession with no equivalent, + although 'and hashish' is a second substance sold in the same transaction + and the total across the two is unstated. A part whose content words are + only substance words is no offence of its own: the amount beside it is held + (quantity_held) and the refusal carries SPLIT_AMOUNT_CAUTION.""" + raw = part or "" + if _AMOUNT.search(raw) or not _plain_chars(raw, digits=True) or fold_erases_word(raw): + return False + folded = set(re.findall(r"[a-z]+", _fold(raw))) + if folded & _UNIT_TOKENS: + return False + if not (folded - STOP - QUANTITY_NEUTRAL_WORDS) - _SUBSTANCE_WORDS: + return False + scored = match_pairings(raw, pairings, state=state) + if not scored: + return False + tokens = _offence_tokens(expand(raw, state)[0]) + if any(ch.isdigit() for t in tokens for ch in t): + return False + seqs = tuple(s for _, p in scored for s in quantity_terms(p)) + return offence_words_only(tokens, seqs) and not _cut_off(raw) + + +SPLIT_AMOUNT_CAUTION = ( + " A part that is not an offence of its own ({parts}) may qualify the amount written " + "beside it: do not enter that amount on its own -- enter the offence without it (for " + "example 'sale of marijuana'), and the quantity is reported open, unless the record " + "states the exact amount for that offence alone.") + + +def split_amount_caution(parts, pairings, state=None): + """The caution a multi-offence refusal adds (verification round G1, Q4) + when one part names an amount and another part is not a plain offence + ('sale of marijuana 2 kg and over' -> '... 2 kg'; 'over'): '' otherwise.""" + if len(parts) < 2 or not any(_AMOUNT.search(x or "") for x in parts): + return "" + # a part that is no plain offence: words beside the offence ('over'), or an + # amount that is no offence of its own ('10 kg seized') + loose = [x for x in parts if not plain_offence(x, pairings, state) + and (not _AMOUNT.search(x or "") or not match_pairings(x, pairings, state=state))] + if not loose: + return "" + return SPLIT_AMOUNT_CAUTION.format(parts="; ".join(f"'{x}'" for x in loose)) + + +# Q6 (verification round G1): simple cannabis possession is curated as having +# no Canadian equivalent within the legal limits. That answer stands only when +# the words show the possession was within them -- no amount at all in plain +# offence words, or ONE exact amount (settled_quantity) of at most the former +# CDSA Schedule VIII amount: 'Cannabis resin 1 g', 'Cannabis (marihuana) 30 g' +# (s. 4(5): summary conviction only 'in an amount that does not exceed the +# amount set out for that substance in Schedule VIII'). A form Schedule VIII +# sets no amount for (SCHEDULE_VII_FORMS) with any amount, a bound, an +# estimate or a larger amount leaves it NOT SETTLED (POSSESSION_LIMIT_QUESTION). +POSSESSION_LIMIT_PAIRINGS = frozenset({"cannabis-possession"}) +SCHEDULE_VIII_KG = {"resin": 0.001, "marihuana": 0.030} +_RESIN_WORDS = frozenset({"hash", "hashish", "resin"}) +POSSESSION_LIMIT_QUESTION = ( + "was the cannabis within the Canadian limits? The record's words do not show it: " + "before 2018-10-17 former CDSA s. 4(5) made possession summary-only only 'in an " + "amount that does not exceed the amount set out for that substance in Schedule VIII' " + "(Cannabis resin 1 g; Cannabis (marihuana) 30 g), and above it s. 4(4) applied " + "(indictable, 'five years less a day', or summary conviction); the Cannabis Act " + "prohibits possessing, in a public place, cannabis 'equivalent to more than 30 g of " + "dried cannabis' (s. 8(1)(a)) and possessing 'any cannabis that they know is illicit " + "cannabis' (s. 8(1)(b)), indictable, 'five years less a day' (s. 8(2)(a)). If neither " + "paragraph of s. 8(1) is made out on the facts, there is no Canadian offence to " + "equate to; if one is, the equivalent is hybrid, five years less a day -- get the " + "amount, the form, where it was possessed and how it was obtained") + + +def possession_limit_open(pairing, offense, context=None): + """True when a POSSESSION_LIMIT_PAIRINGS pairing's 'no equivalent within the + limits' answer is not shown by the record's words (see above), by the facts + or by the disposition (context, quantity_context_open). + + GQ-6 (verification round H1): with NO amount at all, a resin or + concentrate record kept the settled 'no equivalent' -- 'hashish', + 'possession of hashish', 'possession of cannabis resin'. The Canadian line + for resin was 1 g (former CDSA Schedule VIII), which almost any real resin + record exceeds, and the Cannabis Act counts a concentrate against the 30 g + limit as its dried equivalent (Schedule 3: 0.25 g of concentrate to 1 g of + dried cannabis), so the words do not show the possession was within either. + A resin or concentrate word with no amount now answers DEPENDS with + POSSESSION_LIMIT_QUESTION, as an over-limit amount does; plain marijuana + possession with no amount keeps the curated NO (Q6).""" + if (pairing or {}).get("id") not in POSSESSION_LIMIT_PAIRINGS: + return False + raw = offense or "" + if fold_erases_word(raw) or quantity_context_open(quantity_kg(raw), context): + return True + if not _AMOUNT.search(raw): + # no amount: only plain offence words (a statute code or a count the + # matcher reads may carry digits; a bare number may not) + if not _plain_chars(raw, digits=True): + return True + if set(re.findall(r"[a-z]+", _fold(raw))) & _UNIT_TOKENS: + return True + if set(re.findall(r"[a-z]+", _fold(raw))) & (_RESIN_WORDS | SCHEDULE_VII_FORMS): + return True # GQ-6: resin is 1 g, a concentrate counts as its equivalent + tokens = _offence_tokens(expand(raw)[0]) + if any(ch.isdigit() for t in tokens for ch in t): + return True + return (not offence_words_only(tokens, quantity_terms(pairing), SCHEDULE_VII_WORDS) + or _cut_off(raw)) + kg = settled_quantity(pairing, raw, extra=SCHEDULE_VII_WORDS, context=context) + if kg is None or schedule_vii_forms(raw): + return True + words = set(re.findall(r"[a-z]+", _fold(raw))) + limit = SCHEDULE_VIII_KG["resin" if words & _RESIN_WORDS else "marihuana"] + return kg > limit + + +def limb_entries(pairing): + return [l for l in pairing.get("limbs") or [] if isinstance(l, dict) and l.get("key")] + + +def limb_entry(pairing, limb): + return next((l for l in limb_entries(pairing) if l["key"] == limb), None) + + +def limb_label(pairing, limb): + entry = limb_entry(pairing, limb) + return (entry or {}).get("label") or limb or "" + + +def limb_max(pairing, limb=None): + """The current maximum for this limb: its own max_years when the curated + limb names one, else the pairing's.""" + entry = limb_entry(pairing, limb) if limb else None + if entry is not None and "max_years" in entry: + return entry["max_years"] + return pairing.get("max_years") + + +def limb_provision(pairing, limb=None): + """The Canadian provision a curated limb names ('C-46 s. 235'), or None: + what the express exceptions to ss. 463-465 are keyed on (roadmap T2.4).""" + entry = limb_entry(pairing, limb) if limb else None + return (entry or {}).get("provision") + + +def select_limbs(pairing, offense, context=None): + """Which of the pairing's limbs the record can be. context: the facts and + the disposition, read for the quantity only (quantity_context_open). + + Returns {"keys": [limb keys] ([None] for a pairing without limbs), + "open": dimensions the record leaves open ('term': its words select no + single limb; 'quantity': the amount decides between quantity limbs), + "quantity_kg": the amount that settled a quantity limb or None, "terms": + the limb phrases the record named, "forms": the Schedule VII forms named + when the quantity is open, "quantity_why": when it is open, why -- 'words' + (no amount the allowlist accepts), 'held' (quantity_held), 'form' (a + Schedule VII form with 3 kg or less) or 'context' (the facts or the + disposition name another amount, or qualify one)}. + + A limb is selected when one of its phrases occurs in the record's + matching text (folded and expanded as the pairing matcher reads it). + Several limbs selected by different phrases, or none selected, leave the + term open: every candidate limb is kept, and the caller reports each + reading where they give different branches. Limbs that share their + phrases and differ only by quantity are split by an amount written in + the record (settled_quantity, the allowlist); with no amount the reader + is sure of, both are kept.""" + limbs = limb_entries(pairing) + if not limbs: + return {"keys": [None], "open": [], "quantity_kg": None, "terms": [], "forms": [], + "quantity_why": None} + # the matcher drops a drug amount ('sale of 5 kg of marijuana' reads + # 'sale of of marijuana'): the doubled preposition is closed up so the + # limb phrase is found + text = re.sub(r"\b(of|in) (?:of|in)\b", r"\1", f" {expand(offense)[0]} ") + hits, named = [], [] + for l in limbs: + found = [t for t in l.get("terms") or [] + if _fold(t).strip() and f" {_fold(t).strip()} " in text] + if found: + hits.append(l) + named += found + cands = hits or limbs + open_ = [] + if len({tuple(sorted(l.get("terms") or [])) for l in cands}) > 1 or not hits: + if len(cands) > 1: + open_.append("term") + kg, forms, why = None, [], None + qlimbs = [l for l in cands if l.get("quantity") in QUANTITY_LIMBS] + if qlimbs: + # verification round G1: only the allowlist settles (settled_quantity); + # a part of a split string whose other parts are not plain offences + # never settles (Q4, quantity_held); a Schedule VII form never settles + # the 3-kg-or-less limb (Q5). Round H1: an amount in the facts or the + # disposition that the offence text does not state opens it (GQ-5) + forms = schedule_vii_forms(offense) + kg = settled_quantity(pairing, offense, + extra=frozenset(forms) | SCHEDULE_VII_WORDS) + why = "words" if kg is None else None + if kg is not None and quantity_context_open(kg, context): + kg, why = None, "context" + if kg is not None and quantity_is_held(): + kg, why = None, "held" + if kg is not None and forms and QUANTITY_LIMBS["3kg-or-less"](kg): + kg, why = None, "form" + if kg is not None: + cands = [l for l in cands if l.get("quantity") not in QUANTITY_LIMBS + or QUANTITY_LIMBS[l["quantity"]](kg)] + elif len(qlimbs) > 1: + open_.append("quantity") + opened = "quantity" in open_ + return {"keys": [l["key"] for l in cands], "open": open_, "quantity_kg": kg, + "terms": sorted(set(named)), "forms": forms if opened else [], + "quantity_why": why if opened else None} + + +def penalty_at(pairing, offence_date=None, limb=None, strict=False): + """(max_years, note, history_row) for the pairing at the commission date: + the Tran rule (the maximum available when the offence was committed) + applied via the pairing's penalty_history [{"until": "YYYY-MM-DD", + "max_years": n, "summary_only": bool, "note": ..., "limb": key}]. None + max_years = the pairing does not carry a single structured maximum + (value- or schedule-dependent, no equivalent, or a summary-only history + row). limb: the limb the record is (select_limbs); rows for other limbs + are skipped, and after the last row the limb's own current maximum + applies (limb_max). + + A pairing may store penalty_history as null (the held batch of + 2026-09-10 did, on 29 pairings): `or []` treats that as no history + instead of raising TypeError on every dated offence. + + Dates are compared as dates (roadmap T1.1): as strings '2018-6-1' sorted + after '2018-12-17' and '06/01/2022' before it, which flipped the branch. + offence_date may be a date or a YYYY-MM-DD string; anything else raises + ValueError (callers validate first, with parse_date). + + strict: see history_rows -- by default a row whose maximum is HIGHER + than the provision's now reads as the lower current maximum + (methodology (b)); strict=True reads the row as it stands (Tran).""" + od = parse_date(offence_date) + if od: + for until, h in history_rows(pairing, limb, strict): + if od <= until: + return h.get("max_years"), h.get("note", ""), h + return limb_max(pairing, limb), "", None + + +def lenient_since(pairing, h, limb=None): + """True when history row h carries a maximum HIGHER than the provision's + maximum now (limb_max of the row's limb, or of `limb` for a row that + applies to the whole pairing): Canadian law has since become more + lenient. The curated methodology's TIMING rule (b) applies the lower + current maximum in that case; read strictly, Tran applies the maximum at + commission. Summary-only rows and unstructured maxima never compare.""" + mx, cur = h.get("max_years"), limb_max(pairing, h.get("limb") or limb) + return (mx is not None and cur is not None and not h.get("summary_only") + and not pairing.get("summary_only") and mx > cur) + + +def history_rows(pairing, limb=None, strict=False): + """[(until as a date, row)] sorted by date: the rows that apply to the + whole pairing, plus -- when a limb is given -- that limb's own rows. + + A row whose maximum is HIGHER than the provision's now (lenient_since) + is returned, unless strict, as a copy carrying the lower current maximum + (methodology (b): where Canadian law has since become more lenient, the + lower current maximum applies) with "lenient": True and + "at_commission_max": the row's own figure -- so every caller applies + methodology (b) and the maximum at commission is still on the row. + strict=True returns the rows as curated (Tran read strictly).""" + out = [] + for h in pairing.get("penalty_history") or []: + if h.get("limb") not in (None, limb): + continue + if not strict and lenient_since(pairing, h, limb): + h = dict(h, max_years=limb_max(pairing, h.get("limb") or limb), less_a_day=False, + lenient=True, at_commission_max=h["max_years"]) + out.append((parse_date(h["until"]), h)) + return sorted(out, key=lambda t: t[0]) + + +def lenient_rows(pairing, limb=None): + """The rows of this limb (and the whole pairing) whose maximum at + commission is higher than the provision's now (lenient_since), as + curated -- [] for every pairing whose history records increases only.""" + return [h for _, h in history_rows(pairing, limb, strict=True) + if lenient_since(pairing, h, limb)] + + +# ---- point-in-time coverage (roadmap T3.1) -------------------------------- +# CanLex's point-in-time research (tools/pit_history.py) reads Justice Laws +# versions back to 2003-01-01, where its point-in-time record starts; a +# pairing may set a later floor ("pit_floor") where Justice Laws covers its +# Act from a later date (the Income Tax Act: 2004-08-31). +PIT_FLOOR = "2003-01-01" +# {date} reads 'on YYYY-MM-DD' or 'on or before YYYY-MM-DD' (a bound). +PIT_FLOOR_NOTE = ( + "POINT-IN-TIME FLOOR: maximum before {floor} not verified in CanLex -- " + "CanLex's point-in-time research of this provision starts at {floor}, " + "and the offence was committed {date}: an amendment before {floor} would " + "not be in its history. Read the statute as it stood at the offence date " + "before relying on the maximum shown.") + + +def pit_floor(pairing): + return parse_date(pairing.get("pit_floor") or PIT_FLOOR) + + +def not_in_force(pairing, at, limbs=(None,)): + """The curated in_force entries ({"provision", "from", "pit_url", + "limb"}) whose provision came into force AFTER the date `at` -- the + Canadian provision this pairing names did not exist when the offence was + committed (roadmap DECISION (d): what that means is not settled; the + caller prints both readings). An entry with a limb applies only when that + limb is one of the record's possible limbs. [] when `at` is None.""" + at = parse_date(at) + if at is None: + return [] + keys = set(limbs) + return [e for e in pairing.get("in_force") or [] + if parse_date(e["from"]) > at + and (e.get("limb") is None or e.get("limb") in keys)] + + +def tran_bound(pairing, completed=None, convicted=None, limb=None, strict=False): + """(date, label) when the offence date is unknown but a later date bounds + it (roadmap T1.1(2)): offence <= conviction <= sentence completion, so + the offence was committed on or before the earliest bound. When every + maximum the pairing's history gives for dates on or before that bound is + the same, that maximum applied whatever the exact offence date was, and + the bound itself is returned to assess at ('inferred'). When the bound + straddles a change, (None, label) -- the date decides and is not given. + (None, None) when there is no bound or no history. limb, strict: as + penalty_at.""" + bounds = [(d, label) for d, label in ((parse_date(convicted), "conviction"), + (parse_date(completed), "completion")) if d] + rows = history_rows(pairing, limb, strict) + if not bounds or not rows: + return None, None + bound, label = min(bounds) + outcomes = [] + for until, h in rows: + outcomes.append((h.get("max_years"), bool(h.get("summary_only")))) + if bound <= until: + break + else: + outcomes.append((limb_max(pairing, limb), bool(pairing.get("summary_only")))) + if len(set(outcomes)) == 1: + return bound, label + return None, label + + +def tran_outcomes(pairing, completed=None, convicted=None, limb=None, strict=False): + """The maxima the Tran rule could apply when the offence date is NOT + given: one entry per run of equal maxima in the pairing's history, in + date order -- every period, or only those up to the bound a conviction + or completion date sets (offence <= conviction <= completion, roadmap + T1.1(2)). Each entry: {"period", "after", "until", "max_years", + "summary_only", "less_a_day"} ("after"/"until" ISO or None; the period + is 'on or before U', 'after A and on or before U' or 'after A'). [] when + the pairing has no history (for this limb). The caller decides whether + the periods give different branches (a straddled maximum is then + reported, not chosen). limb, strict: as penalty_at.""" + rows = history_rows(pairing, limb, strict) + if not rows: + return [] + bounds = [d for d in (parse_date(convicted), parse_date(completed)) if d] + bound = min(bounds) if bounds else None + runs, prev, bounded = [], None, False + for until, h in rows: + out = [h.get("max_years"), bool(h.get("summary_only")), bool(h.get("less_a_day"))] + if runs and runs[-1][2:] == out: + runs[-1][1] = until + else: + runs.append([prev, until] + out) + prev = until + if bound is not None and bound <= until: + bounded = True + break + if bounded: + runs[-1][1] = None if runs[-1][0] is not None else runs[-1][1] + else: + out = [limb_max(pairing, limb), bool(pairing.get("summary_only")), False] + if runs and runs[-1][2:] == out: + runs[-1][1] = None + else: + runs.append([prev, None] + out) + + def period(after, until): + if after is None and until is None: + return "on any date" # one run: every row equals the current maximum + if after is None: + return f"on or before {until.isoformat()}" + if until is None: + return f"after {after.isoformat()}" + return f"after {after.isoformat()} and on or before {until.isoformat()}" + return [{"period": period(a, u), "after": a.isoformat() if a else None, + "until": u.isoformat() if u else None, "max_years": mx, + "summary_only": so, "less_a_day": lad} for a, u, mx, so, lad in runs] + + +LIFE = 99 # the structured stand-in for imprisonment for life + + +# ---- the maxima a pairing's verify rows quote (wave 2 verifier) ----------- +# A pairing WITHOUT curated limbs carries one structured maximum: its lowest +# provision's, where every provision gives the same branch for the completed +# offence (burglary: 10 years, s. 348(1)(e); a dwelling-house carries life, +# s. 348(1)(d)). ss. 463 and 464 set an attempt's, an accessory's (after the +# fact) and an uncommitted counselled offence's maximum from the OBJECT's, +# and that can cross the 10-year line between the provisions (a dwelling: +# 14 years, s. 463(a); any other place: 5, s. 463(d)(i)). The pairing's own +# verify rows -- statute phrases the curated tests find in the corpus -- +# quote the maxima of the provisions it names; where they quote a higher one +# that the reading's rule puts in a different branch, both are reported as +# two synthetic limbs, never one chosen. +_NUM_WORDS = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, + "eight": 8, "nine": 9, "ten": 10, "eleven": 11, "twelve": 12, "thirteen": 13, + "fourteen": 14} +_STATED_MAX = re.compile(r"imprisonment for life|life imprisonment|(?:not (?:exceeding|more than)|" + r"up to) (\d+|[a-z]+) years?\b(?! less a day)") + + +def maxima_stated(text): + """The maxima a statutory phrase states, as a set of years (LIFE for + imprisonment for life): 'not exceeding ten years' / 'up to ten years' -> + {10} ('up to' since the audit follow-up of 2026-09-21, as + narrative.term_words reads it). A term 'less a day' (a summary maximum) + is not read.""" + out = set() + for m in _STATED_MAX.finditer((text or "").lower()): + if m.group(1) is None: + out.add(LIFE) + continue + g = m.group(1) + n = int(g) if g.isdigit() else _NUM_WORDS.get(g) + if n is not None: + out.add(n) + return out + + +STRUCTURED_LIMB, STATED_LIMB = "structured", "stated-high" +STRUCTURED_LABEL = ("the pairing's structured maximum, {years} (the provision it carries as " + "its lowest)") +STATED_LABEL = ("the higher maximum the pairing's verify row quotes for {act} s. {section}, " + "{years} ('{contains}')") +STATED_OPEN_REASON = ("the pairing carries one structured maximum -- its lowest provision's -- " + "and quotes a higher one for another provision it names; the record's " + "words do not say which provision the conduct is, and ss. 463/464 take " + "the maximum of the object offence") + + +def _years_text(mx): + return "life" if mx >= LIFE else f"{mx:g} years" + + +def stated_limbs(pairing, kinds=(), status="foreign-national"): + """Two synthetic limbs for a pairing WITHOUT curated limbs whose verify + rows quote a maximum higher than its structured max_years, when one of + the inchoate rules `kinds` (INCHOATE_READINGS 'as' values) puts the two + maxima in different branches for this person: [the structured maximum, + the highest one quoted (its verify row named in the label)]. [] for a + pairing with limbs, an unstructured or summary-only maximum, no higher + quote, or no rule that separates them (a party or a conspiracy keeps the + object's own maximum, and the curated branch already holds for it).""" + if (limb_entries(pairing) or pairing.get("max_years") is None + or pairing.get("summary_only") or pairing.get("depends_on")): + return [] + base = pairing["max_years"] + quoted = [(m, v) for v in pairing.get("verify") or [] + for m in maxima_stated(v.get("contains")) if m > base] + if not quoted: + return [] + high, row = max(quoted, key=lambda t: t[0]) + hyb = pairing.get("hybrid") + + def band(mx, kind): + return branch_for(pairing, inchoate_max(kind, mx, False, None, hyb)[0], status) + if not any(band(base, k) != band(high, k) for k in kinds): + return [] + return [{"key": STRUCTURED_LIMB, "terms": [], "max_years": base, + "label": STRUCTURED_LABEL.format(years=_years_text(base))}, + {"key": STATED_LIMB, "terms": [], "max_years": high, + "label": STATED_LABEL.format(act=row["act"], section=row["section"], + years=_years_text(high), contains=row["contains"])}] + + +# ---- a maximum at commission HIGHER than the provision's now (wave 2) ------ +# The curated methodology's TIMING rule (b): Tran governs, and "where +# Canadian law has since become MORE lenient, [IRCC] applies the lower +# current maximum". history_rows applies (b) by default; the maximum at +# commission (a row flagged by lenient_since) is reported beside it where the +# two give different branches for the record (an attempt of former CDSA +# s. 5(3)(a), life: 14 years under s. 463(a), against 7 on today's 14). +LENIENT_LABELS = {"lenient": "the lower maximum now (methodology (b))", + "strict": "the maximum at commission (Tran read strictly)"} +LENIENT_OPEN_REASON = ("the maximum at commission was HIGHER than the provision's maximum now: " + "methodology (b) applies the lower current maximum where Canadian law " + "has since become more lenient, and Tran read strictly applies the " + "maximum at commission -- which governs is not settled in CanLex") +LENIENT_QUESTION = ("which maximum governs where Canadian law has since become more lenient: " + "the lower maximum now (the curated methodology (b)) or the maximum at " + "commission (Tran read strictly)? Not an officer fact -- a question of law " + "CanLex does not settle; the readings give different branches") +LENIENT_NOTE = ("MAXIMUM AT COMMISSION HIGHER THAN NOW: the curated history row for {limb} reads " + "the maximum {when} as {then} ({note}; {pit_url}), and the provision's maximum " + "now is {now}. Methodology (b) applies the lower current maximum where Canadian " + "law has since become more lenient; read strictly, Tran applies the maximum at " + "commission.") + + +# ---------------------------------------------------------------- T2.4 +# Inchoate and party liability (roadmap T2.4). A foreign charge is often an +# attempt, a conspiracy, a solicitation, an accessory or an aiding-and- +# abetting count of another offence. The Criminal Code punishes each by +# reference to the OBJECT offence: s. 463 (an attempt; an accessory after +# the fact), s. 464 (counselling an offence that is not committed: the +# attempt punishment), s. 465(1)(c)/(d) (a conspiracy: the object's +# punishment), ss. 21(1) and 22(1) (a party: the principal's). ss. 463, 464 +# and 465(1) each open 'Except where otherwise expressly provided by law'; +# EXPRESS_EXCEPTIONS lists the provisions the corpus shows for murder. +# The object is matched on its own (inchoate_match); which Canadian +# provision it is stays a curated reading (a limb's 'provision'). + +# (kind, cue) read on the matcher's text (analyse().text: shorthand expanded, +# 'att' -> 'attempted', 'PC 664/187' -> 'attempted pc 187'). At one position +# the longest cue wins ('accessory after the fact' over 'accessory'). +_INCHOATE_CUES = ( + ("accessory", re.compile( + r"\baccessory after the fact(?: (?:to|of))?(?: the commission of)?(?![a-z])")), + ("party", re.compile( + r"\b(?:accessory before the fact(?: (?:to|of))?|aiding (?:and|or) abetting(?: in)?" + r"|aid(?:ed|s)? and abet(?:ted|s)?|aider and abettor(?: (?:to|of|in))?" + r"|accomplice(?: (?:to|of|in))?|party to)(?: the commission of)?(?![a-z])")), + ("accessory-open", re.compile( + r"\baccessory(?: (?:to|of))?(?: the commission of)?(?![a-z])")), + ("conspiracy", re.compile( + r"\b(?:criminal )?(?:conspiracy|conspiring|conspired|conspires|conspire)" + r"(?: (?:to commit|to|of|for))?(?![a-z])")), + ("counselling", re.compile( + r"\b(?:criminal )?(?:solicitation|soliciting|solicited|solicits|solicit" + r"|counsell?ing|counsell?ed|counsels|counsel|incitement|inciting|incited" + r"|incites|incite)(?: (?:to commit|of|to|for))?(?![a-z])")), + ("attempt", re.compile( + r"\b(?:(?:criminal )?attempt(?:ed|ing|s)?(?: (?:to commit|to|of))?|tentativa(?: de)?)" + r"(?![a-z])")), +) + +# The readings of each kind: the Criminal Code rule each applies ('as'). +# Two readings where the record's words leave the role open: a counselled +# offence that was or was not committed (s. 22(1) or s. 464), and a bare +# 'accessory' (before the fact: a party; after the fact: s. 23 / s. 463). +INCHOATE_READINGS = { + "attempt": {"as": "attempt", + "label": "an attempt (C-46 s. 24; punished under s. 463)"}, + "after-the-fact": {"as": "accessory", + "label": "an accessory after the fact (C-46 s. 23; punished " + "under s. 463)"}, + "before-the-fact": {"as": "party", + "label": "an accessory before the fact, a party to the offence " + "(C-46 ss. 21(1), 22(1))"}, + "not-committed": {"as": "counselling", + "label": "counselling an offence that was not committed " + "(C-46 s. 464)"}, + "committed": {"as": "party", + "label": "counselling an offence that was committed, a party to it " + "(C-46 s. 22(1))"}, + "conspiracy": {"as": "conspiracy", "label": "a conspiracy (C-46 s. 465(1))"}, + "party": {"as": "party", "label": "a party to the offence (C-46 s. 21(1))"}, +} +KIND_READINGS = { + "attempt": ("attempt",), "accessory": ("after-the-fact",), + "accessory-open": ("before-the-fact", "after-the-fact"), + "counselling": ("not-committed", "committed"), + "conspiracy": ("conspiracy",), "party": ("party",)} +# The fact that settles a two-reading kind, answered in the facts parameter +# as 'inchoate-role: '. +INCHOATE_FACT = "inchoate-role" +INCHOATE_OPEN_REASON = { + "counselling": ("whether the offence counselled was committed decides between " + "C-46 s. 464 (not committed: the attempt punishment) and s. 22(1) " + "(committed: a party to it, with its maximum)"), + "accessory": ("whether the accessory was before or after the fact decides between " + "C-46 ss. 21(1)/22(1) (a party: the offence's maximum) and s. 23 " + "(after the fact: punished under s. 463)"), +} + +MURDER = "C-46 s. 235" +# 'Except where otherwise expressly provided by law' (ss. 463, 464, 465(1)): +# (rule, object provision) -> the provision that governs instead. Keyed by +# the Canadian provision the curated limb names (a limb's 'provision'), never +# by the foreign word -- which provision a US 'murder' is, is curated. +EXPRESS_EXCEPTIONS = { + ("attempt", MURDER): { + "max_years": LIFE, "section": "C-46 s. 239(1)", + "rule": "attempted murder: imprisonment for life (s. 239(1)(b); (a) and (a.1) " + "add minimum terms where a firearm is used)"}, + ("accessory", MURDER): { + "max_years": LIFE, "section": "C-46 s. 240", + "rule": "an accessory after the fact to murder: imprisonment for life"}, + ("conspiracy", MURDER): { + "max_years": LIFE, "section": "C-46 s. 465(1)(a)", + "rule": "a conspiracy to commit murder: a maximum term of imprisonment for life"}, + ("counselling", MURDER): { + "max_years": LIFE, "section": "C-46 s. 464(a) and s. 239(1)", + "rule": "counselling a murder that is not committed: the punishment for an " + "attempt, which s. 239 sets at imprisonment for life"}, +} + +# Every statutory phrase the rules above rely on, with its corpus chunk +# (tests/test_inchoate.py finds each in data/processed/C-46.json). +INCHOATE_PINS = ( + ("C-46-s21", "Every one is a party to an offence who"), + ("C-46-s21", "for the purpose of aiding any person to commit it"), + ("C-46-s21", "abets any person in committing it"), + ("C-46-s22", "the person who counselled is a party to that offence"), + ("C-46-s22", "counsel includes procure, solicit or incite"), + ("C-46-s23", "An accessory after the fact to an offence is one who"), + ("C-46-s24", "is guilty of an attempt to commit the offence"), + ("C-46-s463", "Except where otherwise expressly provided by law"), + ("C-46-s463", "liable to imprisonment for a term not exceeding fourteen years"), + ("C-46-s463", "one-half of the longest term to which a person who is guilty of " + "that offence is liable"), + ("C-46-s463", "commission of an offence punishable on summary conviction is guilty " + "of an offence punishable on summary conviction"), + ("C-46-s464", "Except where otherwise expressly provided by law"), + ("C-46-s464", "if the offence is not committed"), + ("C-46-s464", "the same punishment to which a person who attempts to commit that " + "offence is liable"), + ("C-46-s465-p1", "Except where otherwise expressly provided by law"), + ("C-46-s465-p1", "the same punishment as that to which an accused who is guilty of " + "that offence would, on conviction, be liable"), + ("C-46-s465-p1", "conspires with any one to commit an offence punishable on summary " + "conviction is guilty of an offence punishable on summary conviction"), + ("C-46-s465-p1", "conspires with any one to commit murder or to cause another person " + "to be murdered"), + ("C-46-s465-p1", "liable to a maximum term of imprisonment for life"), + ("C-46-s235", "shall be sentenced to imprisonment for life"), + ("C-46-s239", "attempts by any means to commit murder"), + ("C-46-s239", "in any other case, to imprisonment for life"), + ("C-46-s240", "accessory after the fact to murder is guilty of an indictable offence " + "and liable to imprisonment for life"), +) + + +# A cheap prefilter for inchoate_match: a pairing term without one of these +# stems cannot read as inchoate (analysing every term on each call is slow; the +# analyse() cache is cleared only when load_aliases installs a different table). +_INCHOATE_STEM = re.compile(r"attempt|\batt\b|accessor|\baid|abet|accomplice|party to|" + r"conspir|solicit|counsel|incit|tentativa", re.I) + + +def _liability_cue(text): + """(kind, match) of the inchoate or party cue in the matcher's text, or + None: the first cue, the longest at one position (_INCHOATE_CUES).""" + best = None + for kind, pat in _INCHOATE_CUES: + m = pat.search(text) + if m and (best is None or (m.start(), -len(m.group(0))) < best[0]): + best = ((m.start(), -len(m.group(0))), kind, m) + return best[1:] if best else None + + +def inchoate(offense): + """The record's words read as inchoate or party liability, or None: + {"kind": attempt | accessory | counselling | conspiracy | party, "cue": + the words, "object": the rest of the charge (the object offence, '' when + the record names none), "readings": KIND_READINGS keys}. Text only: whether + the words name a complete offence instead ('solicitation of prostitution') + is decided against the pairings by inchoate_match.""" + text = analyse(offense).text + best = _liability_cue(text) + if best is None: + return None + kind, m = best + obj = re.sub(r"\s+", " ", text[:m.start()] + " " + text[m.end():]).strip() + return {"kind": "accessory" if kind == "accessory-open" else kind, + "cue": m.group(0).strip(), "object": obj, + "readings": list(KIND_READINGS[kind])} + + +def inchoate_match(offense, pairings, state=None): + """The record read as inchoate or party liability WITH its object offence + matched on its own (roadmap T2.4): inchoate()'s fields plus "scored" (the + object's match_pairings) and "object_pairings" (their ids). None when the + words carry no cue, name no object, or the object matches no pairing -- + and when a pairing term the record names in full uses the cue as part of + a complete offence ('solicitation of prostitution', 'solicitation of a + minor', 'inciting a riot', 'attempt to evade or defeat tax'): a term whose + own object is that same pairing, or no pairing at all. A term that is only + the cue ('solicitation', 'criminal conspiracy') names nothing more.""" + inc = inchoate(offense) + if not inc or not inc["object"]: + return None + scored = match_pairings(inc["object"], pairings, state=state) + if not scored: + return None + text = f" {analyse(offense).text} " + for p in pairings: + for t in _terms(p, state): + if not _INCHOATE_STEM.search(t): + continue # no cue word: the term cannot read as inchoate + # the term read as the record is ('attempt to' -> 'attempted to') + tl = analyse(t).text + if not tl or f" {tl} " not in text: + continue + ti = inchoate(t) + if ti is None or not ti["object"]: + continue + own = match_pairings(ti["object"], pairings, state=state) + if not own or own[0][1]["id"] == p["id"]: + return None + return dict(inc, scored=scored, object_pairings=[p["id"] for _, p in scored]) + + +def inchoate_max(as_kind, max_years, summary_only=False, provision=None, hybrid=None): + """(maximum, rule) for a record read as `as_kind` ('attempt', 'accessory' + (after the fact), 'counselling' (not committed), 'conspiracy', 'party') of + an object offence with this maximum. rule: {"section", "rule", + "exception" (an express exception applied), "changed" (the maximum is + not the object's)}. + + - an express exception for the object's provision governs (murder: ss. + 239, 240, 465(1)(a), 464(a) with 239); + - a party: the principal offence's maximum (s. 21(1)); + - a conspiracy: the object's punishment (s. 465(1)(c)); summary where + the object is summary (s. 465(1)(d)); + - an attempt or an accessory after the fact (s. 463), and counselling an + offence not committed (s. 464: the attempt punishment): life -> 14 + years (s. 463(a)); 14 years or less -> one-half (s. 463(b), or (d)(i) + for a hybrid); a summary-only offence stays summary (s. 463(c), + s. 464(b)). None stays None.""" + exc = EXPRESS_EXCEPTIONS.get((as_kind, provision)) if provision else None + if exc: + return exc["max_years"], {"section": exc["section"], "rule": exc["rule"], + "exception": True, + "changed": max_years != exc["max_years"]} + if as_kind == "party": + return max_years, {"section": "C-46 s. 21(1)", + "rule": "a party is liable to the offence's own maximum", + "exception": False, "changed": False} + if as_kind == "conspiracy": + if summary_only: + return max_years, {"section": "C-46 s. 465(1)(d)", + "rule": "a conspiracy to commit a summary offence is summary", + "exception": False, "changed": False} + return max_years, {"section": "C-46 s. 465(1)(c)", + "rule": "the same punishment as the object offence", + "exception": False, "changed": False} + via = "C-46 s. 464(a) and " if as_kind == "counselling" else "C-46 " + if summary_only: + return max_years, {"section": ("C-46 s. 464(b)" if as_kind == "counselling" + else "C-46 s. 463(c)"), + "rule": "summary conviction, as the object offence", + "exception": False, "changed": False} + if max_years is None: + return None, {"section": via + "s. 463", + "rule": "s. 463 applies to the object's maximum, which is not " + "structured here", "exception": False, "changed": False} + if max_years >= LIFE: + return 14, {"section": via + "s. 463(a)", + "rule": "the object carries life: 14 years", + "exception": False, "changed": True} + para = {True: "s. 463(d)(i)", False: "s. 463(b)"}.get(hybrid, "s. 463(b)/(d)(i)") + return max_years / 2, {"section": via + para, + "rule": "one-half of the object's longest term", + "exception": False, "changed": True} + + +def attempt_max(max_years, summary_only=False): + """The maximum for an ATTEMPT under Criminal Code s. 463 (inchoate_max + for 'attempt', no express exception): life -> 14 years (s. 463(a)); 14 + years or less -> one-half (s. 463(b)); a summary-only offence stays + summary (s. 463(c)); a hybrid offence takes half its indictable term + (s. 463(d)). None stays None.""" + return inchoate_max("attempt", max_years, summary_only)[0] + + +def branch_for(pairing, max_years, status="foreign-national", + summary_only=None): + """The IRPA s. 36 branch as a code: 'serious-criminality', 'criminality', + 'criminality-two-plus-only' (summary-only equivalent), 'none' (no + federal equivalent), 'depends' (value/schedule/facts decide) -- and for + a permanent resident every sub-10-year outcome collapses to + 'not-inadmissible-pr', because s. 36(2) (criminality, including its + two-summary-offences limb) reaches foreign nationals only.""" + if pairing.get("equivalent") is False: + return "none" + if pairing.get("depends_on") and max_years is None: + return "depends" + so = pairing.get("summary_only") if summary_only is None else summary_only + if status == "permanent-resident" and (so or (max_years is not None + and max_years < 10)): + return "not-inadmissible-pr" + if so: + return "criminality-two-plus-only" + if max_years is None: + return "unknown" + if max_years >= 10: + return "serious-criminality" + return "criminality" + + +_BRANCH_TEXT = { + "serious-criminality": "SERIOUS CRIMINALITY -- IRPA s. 36(1)(b) (10 years " + "or more; deemed rehabilitation unavailable)", + "criminality": "CRIMINALITY -- IRPA s. 36(2)(b) (foreign nationals only; " + "a permanent resident is not inadmissible on this alone)", + "criminality-two-plus-only": "summary-only Canadian equivalent: a single " + "conviction grounds nothing; two or more " + "convictions not arising from a single " + "occurrence reach s. 36(2)(b) (foreign " + "nationals only)", + "none": "NO INADMISSIBILITY on the conviction branch: no Canadian " + "federal offence to equate to", + "depends": "DEPENDS -- the branch turns on the facts the pairing names " + "(value, substance schedule, or the elements proved)", + "not-inadmissible-pr": "criminality-level or summary-only equivalent: a " + "PERMANENT RESIDENT is not inadmissible under " + "s. 36(2) (foreign nationals only), and the " + "two-summary-offences limb does not reach a " + "permanent resident either", + "unknown": "not tabulated -- read the pairing's penalty line", + "tran-open": "NOT SETTLED -- the maximum changed over time and the " + "offence date, which decides it (Tran: the maximum at " + "commission governs), was not given; each period's branch " + "is shown", + "limb-open": "NOT SETTLED -- the record's words (or the quantity) do not " + "say which Canadian provision it is (one of the pairing's " + "provisions, or the rule for a counselled offence or an " + "accessory), and their maxima give different branches for " + "this record; each reading is shown", +} + +# Branch codes that report several readings and no band (the offence date +# and/or the limb decides): every consumer treats them alike. +OPEN_BRANCHES = ("tran-open", "limb-open") + + +def branch_text(code): + return _BRANCH_TEXT.get(code, code) + + +def is_attempt(offense): + """True when the record's words read as an attempt -- in its own words + ('attempted', 'attempt to commit', 'tentativa de'), or once its statute + code is read ('PC 664/211' -> 'attempted robbery'). Text only: step 2 + applies s. 463 through inchoate_match, which also reads the object.""" + return (inchoate(offense) or {}).get("kind") == "attempt" + + +# The statute summary printed for each kind (roadmap T2.4). +_INCHOATE_BASE = { + "attempt": ("ATTEMPT: the record says attempted. Criminal Code s. 463 sets " + "the maximum for an attempt at 14 years where the completed " + "offence carries life (s. 463(a)), one-half of the longest term " + "where the completed offence carries 14 years or less " + "(s. 463(b)), a summary conviction where the completed offence is " + "summary-only (s. 463(c)), and one-half of the indictable term for " + "a hybrid offence (s. 463(d))."), + "accessory": ("ACCESSORY: an accessory AFTER the fact (C-46 s. 23) is punished " + "under s. 463, like an attempt: 14 years where the offence carries " + "life, one-half of its longest term otherwise, summary where it is " + "summary-only. An accessory BEFORE the fact -- one who aids, abets " + "or counsels an offence that is committed -- is a party to it " + "(ss. 21(1), 22(1)) with its own maximum. A bare 'accessory' does " + "not say which."), + "counselling": ("COUNSELLING: 'counsel' includes procure, solicit or incite " + "(C-46 s. 22(3)). Counselling an offence that is not committed " + "takes the punishment for an attempt (s. 464(a); summary where the " + "offence is summary, s. 464(b)); if the offence is committed, the " + "person who counselled is a party to it (s. 22(1)), with its own " + "maximum. CAVEAT: some US solicitation statutes are complete " + "offences -- the solicitation is the crime whether or not the " + "offence solicited is committed, and the record may not say which. " + "s. 464 applies only 'if the offence is not committed': both " + "readings are given where they give different branches."), + "conspiracy": ("CONSPIRACY: a conspiracy to commit an indictable offence carries " + "the same punishment as that offence (C-46 s. 465(1)(c)); one to " + "commit a summary offence is summary (s. 465(1)(d)). The object " + "offence is screened for its maximum."), + "party": ("PARTY: every one who actually commits, aids or abets an offence is a " + "party to it (C-46 s. 21(1)) and liable to its own maximum -- Canadian " + "law does not grade an accomplice below the principal."), +} +EXPRESS_EXCEPTIONS_NOTE = ( + "EXPRESS EXCEPTIONS: ss. 463, 464 and 465(1) apply 'Except where otherwise " + "expressly provided by law'. For murder, an attempt (s. 239), an accessory after " + "the fact (s. 240), a conspiracy (s. 465(1)(a)) and counselling a murder that is " + "not committed (s. 464(a), through s. 239) all carry imprisonment for life. " + "CanLex applies these only where the curated pairing marks the object as murder " + "(a limb naming C-46 s. 235).") +# The approved baseline has no murder limb (held): a record whose object +# words name murder says what the express exceptions would give. +MURDER_HELD_NOTE = ( + "MURDER NOT MARKED IN THIS DEPLOYMENT: the record's words name murder, but the curated " + "pairing carries no limb marking the object as murder (C-46 s. 235; that limb is held " + "for review), so the maximum shown is the general rule ({section}). If the object is " + "murder, the express exception governs: {exception} -- imprisonment for life. The branch " + "is serious criminality either way.") +MURDER_EXCEPTION_TEXT = { + "attempt": "an attempt to commit murder, s. 239(1)(b)", + "accessory": "an accessory after the fact to murder, s. 240", + "counselling": "counselling a murder that is not committed, s. 464(a) through s. 239(1)(b)", +} + + +def murder_held_note(inc, pairing, section): + """MURDER_HELD_NOTE for an attempt, accessory (after the fact) or + counselling record whose object words name murder, matched to a pairing + with no limb naming C-46 s. 235 (the approved baseline); '' otherwise.""" + if not inc or inc.get("kind") not in MURDER_EXCEPTION_TEXT: + return "" + if not re.search(r"\bmurder", inc.get("object") or ""): + return "" + if any(l.get("provision") == MURDER for l in limb_entries(pairing)): + return "" + return MURDER_HELD_NOTE.format(section=section or "C-46 s. 463", + exception=MURDER_EXCEPTION_TEXT[inc["kind"]]) + + +def inchoate_note(kind, max_years=None, summary_only=False): + """The statute summary for a record read as `kind` (inchoate()), with the + express exceptions where they can apply; for an attempt with a maximum, + the adjusted figure. '' for anything else.""" + base = _INCHOATE_BASE.get(kind) + if not base: + return "" + if kind in ("attempt", "accessory", "counselling", "conspiracy"): + base += " " + EXPRESS_EXCEPTIONS_NOTE + if kind == "attempt" and max_years is not None: + adj = attempt_max(max_years, summary_only) + if adj != max_years: + shown = "life" if max_years >= LIFE else f"{max_years:g} years" + base += (f" Here: {shown} attempted equates to {adj:g} years -- " + f"{'still serious criminality' if adj >= 10 else 'criminality, not serious criminality'}; " + "the branch line and the structured result already apply it.") + return base + + +def attempt_note(offense, max_years=None, summary_only=False): + """Inchoate liability changes the maximum (Criminal Code ss. 463-465): + the note for the record's kind (inchoate_note), with the adjusted figure + for an attempt; '' when the words carry no inchoate cue.""" + inc = inchoate(offense) + return inchoate_note(inc["kind"], max_years, summary_only) if inc else "" + + +def result_line(**fields): + """The machine-readable hand-off appended to every screening tool's + output: one line, stable keys, so the next step consumes exact values + instead of a transcription.""" + return "CANLEX_RESULT: " + json.dumps(fields, ensure_ascii=False, + sort_keys=True) + + +# ---- the shared confidence block (roadmap T4.2) ------------------------- +# Every screening tool's CANLEX_RESULT carries 'confidence': what each step's +# answer rests on, so a client model does not read a no-authority pairing +# with the same confidence as a judicially considered one. The values are +# read from the curated records (their 'status'), the state row's +# provenance tier, the authority registry and the maximum actually applied. + +def confidence_block(step1=None, step2=None, step3=None): + """{"step1": [...], "step2": [...], "step3": {...}} -- a list per step + (one item per entry or pairing the answer rests on; a tie has two), None + for a step the tool does not take.""" + return {"step1": list(step1) if step1 is not None else None, + "step2": list(step2) if step2 is not None else None, + "step3": step3} + + +def step1_confidence(entry, row=None, n=None): + """Step 1: the disposition entry's curated authority status and whether + the state row the verdict came from was verified against official text + (True), is a row that was not (False), or no state row applied (None).""" + out = {"entry": (entry or {}).get("id"), + "entry_status": (entry or {}).get("status"), + "row_verified": None if not row else bool(row.get("verified")), + "row_tier": (None if not row else "verified" if row.get("verified") else + "not-verified" if row.get("unverifiable") else "secondary-unverified")} + if n is not None: + out["n"] = n + return out + + +def authorities_in_corpus(pairing): + """'n/m': of the m decisions the pairing cites, how many the authority + registry holds with their text in the corpus. None when it cites none; + '?/m' when the registry is not in this deployment, or cannot be read -- + which one stderr line says, naming the error's type (audit follow-up + 2026-09-21: every failure was swallowed silently).""" + cites = [a.get("cite") for a in (pairing or {}).get("authorities") or [] + if isinstance(a, dict) and a.get("cite")] + if not cites: + return None + try: + from canlex.authorities import load, lookup + reg = load() + if not reg: + return f"?/{len(cites)}" + n = sum(1 for c in cites if (lookup(c, reg)[1] or {}).get("status") == "in-corpus") + except (OSError, ValueError, KeyError, TypeError) as exc: + print(f"CanLex MCP: the authority registry could not be read ({type(exc).__name__}); " + f"authorities_in_corpus reads ?/{len(cites)}", file=sys.stderr) + return f"?/{len(cites)}" + return f"{n}/{len(cites)}" + + +def maximum_source(pairing, fields): + """Where the maximum the answer applied comes from: 'current' (the + provision's maximum now), 'history' (a curated penalty-history row -- + the maximum at commission, Tran), 'timeline' (a history row checked + against a Justice Laws point-in-time page, pit_url), 'limb' (the + current maximum of the limb the record selected), 'resolver' (an + element answer selected the provision). None when no one maximum is + applied (a tie, an open date or limb, the elements undecided).""" + if not pairing or not fields: + return None + if (fields.get("branch") in ("tie", "depends", "unknown") + OPEN_BRANCHES + or fields.get("max_years") is None): + return None + if fields.get("resolved_by") == "elements": + return "resolver" + if fields.get("resolved_by") in ("value", "schedule"): + return "current" + at = fields.get("assessed_at") + try: + at = parse_date(at) if at and at != "current" else None + except ValueError: + at = None + limb = fields.get("limb") + if limb and not limb_entry(pairing, limb): + limb = None + _, _, hist = penalty_at(pairing, at, limb) + if hist is not None: + return "timeline" if hist.get("pit_url") else "history" + return "limb" if limb else "current" + + +def step2_confidence(pairing, fields=None, n=None): + """Step 2: the pairing's curated authority status, how many of the + decisions it cites are in the corpus, and where the maximum applied + comes from.""" + out = {"pairing": (pairing or {}).get("id"), + "authority_status": (pairing or {}).get("status"), + "authorities_in_corpus": authorities_in_corpus(pairing), + "maximum_source": maximum_source(pairing, fields)} + if n is not None: + out["n"] = n + return out + + +def step3_confidence(limbs=None, limbs_tested=False): + """Step 3: how many IRPR s. 18(2) / s. 18.1 limbs the answer left UNKNOWN + (None when the tool does not test the limbs one by one -- the single-entry + tools read an other-record flag instead).""" + return {"limbs_tested": bool(limbs_tested), + "limbs_unknown": (sum(1 for l in limbs or [] if l.get("status") == "unknown") + if limbs_tested else None)} + + +# ---- the sentence as imposed, and completion IF SERVED AS IMPOSED (T4.5) -- +# A rap sheet gives the sentence imposed and its date, rarely the date the +# whole sentence was completed. Guide 5312 (IRCC, the rehabilitation +# application guide; corpus chunk policy-imm-5312-12, 'Calculating the five +# (5) year waiting period') says, for each type of sentence, when the five +# years start. CanLex uses those rules to project the completion date IF the +# sentence was served exactly as imposed -- never as a finding: early +# termination, revocation, an extension or a late fine payment move it, so +# the projection is not a bound in either direction and gives no verdict. +# The quotations are verbatim; tests/test_decisive_card.py pins them. +GUIDE_5312_CHUNK = "policy-imm-5312-12" +GUIDE_5312_RULES = { + "suspended": "Suspended sentence: count five (5) years from the date of sentencing.", + "fine": ("Suspended sentence with a fine: count five (5) years from the date the " + "fine was paid."), + "imprisonment": ("Imprisonment without parole: count five (5) years from the end of " + "the term of imprisonment."), + "parole": "Imprisonment and parole: count five (5) years from the completion of parole.", + "probation": ("Probation: probation is part of the sentence. Count five (5) years from " + "the end of the probation period."), + "driving": "Driving prohibition: count five (5) years from the end date of the prohibition.", +} +GUIDE_5312_COURT = "You are prohibited by the Criminal Court from driving." +IF_SERVED_LABEL = ("IF SERVED EXACTLY AS IMPOSED (not a bound: early termination, " + "revocation, extension or late fine payment move it)") +MARGIN_MONTHS = 12 + +# (the code audit of 2026-09-21: a comma between digits followed by exactly three digits +# is a thousands separator -- '$1,500 fine' is one part -- and every other comma splits, +# '2 yrs prob,180 days jail' included; before this '$1,500 fine' split into a $1 fine and +# an unread '500 fine' and the projection was blocked for the wrong reason) +_SENT_SPLIT = re.compile(r"\s*(?:;|(? (2 if suspended else 1): + return None + (n, unit), served = durs[0], None + if suspended: + if len(durs) == 2: + if durs[1][1] != unit or durs[1][0] > n: + return None + served = n - durs[1][0] + else: + served = 0 + return {"kind": kind, "text": t, "n": n, "unit": unit, "suspended": suspended, + "served_n": served, "amount": None} + + +def parse_sentence(text): + """The parts of a sentence as a rap sheet writes it ('3 YRS PROB', + '30 DAYS JAIL SUSP', '$500 FINE', '12 MOS DL SUSP', '180 DAYS JAIL, 170 + SUSP'): {"parts": [...], "unread": [text CanLex cannot read]}. A bare + 'N SUSP' after a term of imprisonment is the part of it suspended.""" + parts, unread = [], [] + for raw in _SENT_SPLIT.split(text or ""): + raw = raw.strip() + if not raw: + continue + m = _SENT_BARE.match(raw) + prev = parts[-1] if parts else None + if m and prev and prev["kind"] == "imprisonment" and not prev["suspended"]: + k = float(m.group(1)) + if k <= prev["n"]: + prev.update(suspended=True, served_n=prev["n"] - k, + text=f"{prev['text']}, {raw}") + continue + part = _sentence_part(raw) + if part is None: + unread.append(raw) + else: + parts.append(part) + return {"parts": parts, "unread": unread} + + +def add_months(date, months): + """`date` moved by whole months: the day with the same number, 'or, if + that month has no day with that number, the last day of that month' + (Interpretation Act s. 28(c), as add_years).""" + import calendar + y, m = divmod(date.month - 1 + int(months), 12) + y += date.year + return date.replace(year=y, month=m + 1, + day=min(date.day, calendar.monthrange(y, m + 1)[1])) + + +def _add(date, n, unit): + """date + n units; None when n is not a whole number of months or years.""" + if unit == "days": + return date + _dt.timedelta(days=round(n)) + if unit == "weeks": + return date + _dt.timedelta(days=round(n * 7)) + months = n if unit == "months" else n * 12 + if months != int(months): + return None + return add_months(date, int(months)) + + +def projected_completion(sentence_imposed, sentence_date): + """The completion date IF the sentence was served exactly as imposed + (roadmap T4.5), from Guide 5312's rules (GUIDE_5312_RULES): the end of + the part that ends last. Returns {"date", "rule", "part", "blocked", + "if_administrative", "parts"}: blocked names why no date is projected (a + fine -- the five years run from the day it was paid, which the record + does not show; a part CanLex cannot read). A driving prohibition ends + the sentence only if a criminal court imposed it (Guide 5312: 'You are + prohibited by the Criminal Court from driving'); where it is the last + part to end, if_administrative is the date without it (both readings + are printed).""" + parsed = parse_sentence(sentence_imposed) + out = {"date": None, "rule": None, "part": None, "blocked": None, + "if_administrative": None, "parts": parsed["parts"]} + if sentence_date is None: + out["blocked"] = "no sentence date" + return out + if parsed["unread"]: + out["blocked"] = ("a part of the sentence CanLex cannot read: '" + + "', '".join(parsed["unread"]) + "' -- when it ends is not known") + return out + if not parsed["parts"]: + out["blocked"] = "no sentence given" + return out + fines = [p for p in parsed["parts"] if p["kind"] == "fine"] + if fines: + out["blocked"] = ("a fine ('" + "', '".join(p["text"] for p in fines) + "'): Guide " + "5312 counts the five years from the day it was paid -- '" + + GUIDE_5312_RULES["fine"] + "' -- and the record does not show " + "that it was paid, or when") + return out + ends = [] + for p in parsed["parts"]: + if p["kind"] == "imprisonment" and p["suspended"] and not p["served_n"]: + end, rule = sentence_date, "suspended" + elif p["kind"] == "imprisonment": + end = _add(sentence_date, p["served_n"] if p["suspended"] else p["n"], p["unit"]) + rule = "imprisonment" + else: + end, rule = _add(sentence_date, p["n"], p["unit"]), p["kind"] + if end is None: + out["blocked"] = (f"'{p['text']}': a term that is not a whole number of months " + "or years -- its end date is not computed") + return out + ends.append((end, rule, p)) + last = max(ends, key=lambda e: e[0]) + out.update(date=last[0], rule=last[1], part=last[2]["text"]) + if last[1] == "driving": + rest = [e for e in ends if e[1] != "driving"] + out["if_administrative"] = max(e[0] for e in rest) if rest else sentence_date + if out["if_administrative"] == out["date"]: + out["if_administrative"] = None + return out + + +def within_margin(date, today, months=MARGIN_MONTHS): + """True when `date` lies within `months` of `today`, either side.""" + return date is not None and add_months(today, -months) <= date <= add_months(today, months) + + +# The one date format every screening tool accepts: year first, anchored at +# both ends. Month and day may be one or two digits ('2018-6-1'); US +# 'MM/DD/YYYY' and anything trailing ('2018-06-011') are refused, because a +# date read the wrong way silently flips the Tran maximum (roadmap T1.1). +_DATE = re.compile(r"^\s*(\d{4})-(\d{1,2})-(\d{1,2})\s*$") + + +def parse_date(text): + """A datetime.date from a YYYY-MM-DD string (or a date), None for an + empty value; ValueError for anything else, including impossible dates + ('2018-02-30').""" + if text is None or isinstance(text, _dt.date): + return text.date() if isinstance(text, _dt.datetime) else text + s = str(text) + if not s.strip(): + return None + m = _DATE.match(s) + if not m: + raise ValueError(f"{s.strip()!r} is not a YYYY-MM-DD date") + y, mo, d = (int(g) for g in m.groups()) + # (the code audit of 2026-09-21: a year the ten-year clocks cannot be added to -- + # datetime stops at 9999 -- raised a bare ValueError from add_years past every + # tool's date guard; the guard now answers, as for any other unreadable date) + if not DATE_YEAR_MIN <= y <= DATE_YEAR_MAX: + raise ValueError(f"{s.strip()!r}: the year must be between {DATE_YEAR_MIN} and " + f"{DATE_YEAR_MAX}") + return _dt.date(y, mo, d) + + +DATE_YEAR_MIN = 1000 +DATE_YEAR_MAX = 2999 + + +def years_since(start, today): + """Whole years from `start` itself to `today`: the largest n with + add_years(start, n) <= today (0 when today precedes start). Counting + through add_years keeps every elapsed figure consistent with the dates + the tools print, 29 February included (roadmap T1.12).""" + if today < start: + return 0 + n = today.year - start.year + while n > 0 and add_years(start, n) > today: + n -= 1 + return n + + +def full_years(since, today): + """Full years elapsed from the day after `since` to `today` -- the + IRPR s. 18(2) / s. 18.1 clock ('since the day after the completion').""" + return years_since(since + _dt.timedelta(days=1), today) + + +def add_years(date, years): + """`date` moved by whole years. A year is 'any period of twelve + consecutive months' (Interpretation Act s. 37(1), I-21-s37), and a period + of months after a specified day ends on the day with the same number 'or, + if that month has no day with that number, the last day of that month' + (s. 28(c), I-21-s28): five years after 29 February 2016 is 28 February + 2021. Both phrases are pinned by tests/test_rehab_properties.py.""" + try: + return date.replace(year=date.year + years) + except ValueError: # 29 February -> last day of February + return date.replace(year=date.year + years, day=28) + + +def date_warnings(offence_date=None, completed=None, today=None): + """Sanity flags for the dates a record carries: anything in the future + or out of order is reported, never silently computed on.""" + today = today or _dt.date.today() + out = [] + if offence_date and offence_date > today: + out.append(f"the offence date {offence_date} is after the assessment " + f"date {today}") + if completed and completed > today: + out.append(f"the sentence-completion date {completed} is after the " + f"assessment date {today} -- the sentence is not complete " + "and no clock has started") + if offence_date and completed and completed < offence_date: + out.append(f"the sentence-completion date {completed} precedes the " + f"offence date {offence_date}") + return out + + +# ---- disposition cues ------------------------------------------------- + +# Words that only a conviction carries. A record that says any of these +# and none of the non-conviction phrases below IS a conviction, whatever +# the curated entries know. +_CONVICTION_CUES = re.compile( + r"\b(convicted|conviction|found guilty|adjudged guilty|adjudicated guilty|" + r"plea(?:d|ded)? guilty|pled guilty|guilty plea|plea of guilty|" + r"judgment of conviction|judgement of conviction|sentenced|sentence of|" + r"fined|fine of|jail|prison|imprisonment|verdict of guilty|guilty verdict|" + r"jury verdict|guilty|" + # the review of batch 1, round 5: a sentence term ('2 years tdcj', '30 days jail', '2 years + # probation'), a bare 'probation' that no programme word qualifies (never 'pretrial + # probation', 'juvenile probation', 'probation before judgment'), and a case 'disposed of + # by plea' (a plea of guilty unless a nolo or not-guilty word sits beside it) + r"\d+\s*(?:years?|months?|days?)\s+(?:in\s+)?(?:tdcj|doc|idoc|adc|bop|hoc|jail|prison|" + r"probation|community supervision)|" + r"(? + 'convicted', 'DEF ADJ' -> 'deferred adjudication').""" + t = (disposition or "").lower() + for pat, full in _DISPOSITION_ABBREVIATIONS: + t = pat.sub(full, t) + return t + + +# A decided appeal that left the conviction standing (roadmap T2.7, audit): +# 'convicted, appeal dismissed' and 'conviction affirmed on appeal' are +# convictions -- not a dismissal of the charge, not a pending appeal. The +# phrases are removed before the cues and the curated entries read the words; +# the conviction words stay. 'Dismissed on appeal' (the charge dismissed by the +# appeal court) is not one of them. A prosecution appeal dismissed leaves an +# acquittal standing, and the acquittal words stay too. +APPEAL_UPHELD = re.compile( + # (the review of batch 1, round 5: the appeal may carry its object -- 'APPEAL AGAINST + # CONVICTION DISMISSED', 'APPEAL AGAINST CONVICTION AND SENTENCE DISMISSED' -- and a label + # separator -- 'APPEAL: DISMISSED', 'APPEAL - DISMISSED', 'DISMISSED - APPEAL'; 'APPEAL NOT + # ALLOWED', an appeal allowed as to the sentence only and an appeal allowed with the + # sentence varied leave the conviction standing too) + r"\b(?:(?:the |his |her |their )?appeals?(?: against (?:the |his |her |their )?" + r"(?:convictions?(?: and sentences?)?|sentences?|verdicts?))?\s*(?:[:\-–—]+\s*)?" + r"(?:was |were |is |has been |have been )?" + r"(?:dismissed|denied|refused|rejected|abandoned|withdrawn|not allowed)|" + r"(?:dismissed|denied|refused)\s*(?:[:\-–—]+\s*)?appeals?\b(?!\s+(?:pending|filed|lodged|noted|" + r"taken|by))|" + r"(?:dismiss(?:ed|ing)|den(?:ied|ying)|refus(?:ed|ing)) (?:the |his |her |their )?" + r"appeals?|(?:affirmed|upheld|confirmed) on appeal|" + r"appeals? (?:was |were |is |has been )?allowed (?:as to|on|in respect of|regarding|re) " + r"(?:the |his |her )?sentenc\w*(?: only)?|" + r"appeals? (?:was |were |is |has been )?allowed\b[^;,|]{0,40}?\b(?:sentence|term|penalty) " + r"(?:reduced|varied|substituted|remitted)|" + # the review of batch 1, round 4: an appeal against SENTENCE allowed leaves the + # conviction standing, and 'no further action on appeal' ends the appeal + r"appeals? against (?:the |his |her |their )?sentences? (?:was |were |is |has been )?" + r"(?:allowed|succeeded|successful|upheld|granted)|no further action on appeal)\b") +CONVICTION_UPHELD = re.compile( + r"\b(conviction|judgment|judgement|sentence) (?:was |is |has been )?" + r"(?:affirmed|upheld|confirmed)\b") + + +def strip_appeal_upheld(text): + """The words with a decided appeal that left the conviction standing + removed ('convicted, appeal dismissed' -> 'convicted, '); an appeal against the + conviction dismissed keeps the word 'conviction' (round 5: 'APPEAL AGAINST CONVICTION + DISMISSED' alone is the conviction standing).""" + return CONVICTION_UPHELD.sub(r"\1", APPEAL_UPHELD.sub( + lambda m: " conviction " if "conviction" in m.group(0) else " ", text)) + + +# ---- the record-order cue table (disposition roadmap T1.6, critic items) ----- +# ONE table of record-wording cues, read in record order before the +# disposition cues and the curated entries see the words. T4.1's decisive +# facts reuse it: never copy these patterns elsewhere. Each pattern reads the +# lower-cased, accent-folded record words WITH their punctuation (the +# segment breaks ';', ',', '|', ':' and a spaced dash matter). Two kinds of +# rule use it (record_order): +# - routing rules rewrite the words the cues and entries read: a not-guilty +# PLEA is not a verdict; 'trial set' is a pending charge; a programme +# revoked and guilt then adjudicated reads from the adjudication (the later +# judgment wins -- the revocation must be bound to a programme word, and a +# reinstated / modified / continued / extended programme blocks it); a +# juvenile tried in adult court is not read as a juvenile adjudication; a +# refused pardon, expungement, sealing, set-aside, vacatur, programme or +# motion is not relief; a stayed jail term or fine is not a pending charge; +# an unreported disposition is not a pending charge; +# - question rules leave the words and raise a flag the server answers +# NOT SETTLED with its deciding question (DISPOSITION_FLAG_LINES): counts +# with different outcomes, guilt found but sentence not imposed, conviction +# words then a dismissal, a set-aside with no mechanism named. +# The programme words the revocation rules bind to (the 'programme' cue), shared with the +# judgment-upon-revocation cue (round 4: the deferred-sentence names, a deferred +# disposition, a youthful-offender status and a bare 'deferred' are programmes too). +_PROGRAMME_WORDS = ( + r"probation before judg(?:e)?ment|probation|community supervision|supervision|" + r"deferred adjudication|adjudication deferred|deferred judg(?:e)?ment|" + r"deferred entry of judg(?:e)?ment|pbj|adjudication withheld|withheld adjudication|" + r"withhold(?:ing)? of adjudication|pretrial diversion|pre-trial diversion|" + r"diversion|pretrial intervention|pre-trial intervention|pti|ard|acd|acod|cwof|" + r"sis|suspended imposition(?: of sentence)?|plea in abeyance|dej|" + r"withheld judg(?:e)?ment|judg(?:e)?ment withheld|deferred prosecution(?: agreement)?|" + r"deferred sentenc(?:e|ing)|sentence deferred|deferral of sentence|deferred disposition|" + r"youthful offender(?: status| treatment| adjudication| finding)?|yo status|deferred") +DISPOSITION_CUE_TABLE = { + # a not-guilty PLEA (with its NGRI form), never a verdict + "plea-not-guilty": ( + r"\b(?:(?:pled|pleaded|pleads|plead|pleading|entered (?:a )?plea(?: of)?|" + r"plea(?: entered)?)\s*[-:=]?\s*(?:of\s+)?not guilty" + r"(?:\s+by reason of (?:insanity|mental disease\w*(?: or defect)?))?" + r"(?:\s+plea)?(?:\s+(?:entered|withdrawn))?|" + r"not guilty plea(?:\s+(?:entered|withdrawn))?|" + r"plea of not guilty(?:\s+(?:entered|withdrawn))?|" + # the review of batch 1, round 3: 'SET FOR TRIAL; ENTERED NOT GUILTY' is a plea + # (never a jury's or court's entry, never a verdict) + r"(? DISMISSED' with no dash) + "bare-conviction-offence": ( + r"^\s*(?:(?:guilty|convicted)\s*(?:[-:\u2013\u2014]\s*)?\(?(?P[a-z][^;,|:()\[\]]*?)\)?" + r"(?:\s*[-:\u2013\u2014]\s*(?:\$|\d|fine|fined|jail|prison|probation|sentenced|sentence|time served|" + r"costs|community service|suspended|confinement|custody)[^;,|]*)?|" + r"(?P[a-z][^;,|:()\[\]]*?)\s*[-:\u2013\u2014]\s*(?:guilty|convicted|(?:pled|pleaded|plead)\s+guilty|" + r"guilty\s+plea|plea\s+of\s+guilty)" + r"(?:\s*[-:\u2013\u2014]\s*(?:\$|\d|fine|fined|jail|prison|probation|sentenced|sentence|time served|" + r"costs|community service|suspended|confinement|custody)[^;,|]*)?)\s*$"), + "bare-outcome-offence": ( + r"^\s*(?:(?:not guilty|acquitted|dismissed|nolle\w*|nol pros\w*|dropped)\s*(?:[-:\u2013\u2014]\s*)?" + r"\(?(?P[a-z][^;,|:()\[\]]*?)\)?|(?P[a-z][^;,|:()\[\]]*?)\s*[-:\u2013\u2014]\s*(?:not guilty|acquitted|" + r"dismissed|nolle\w*|nol pros\w*|dropped)|" + r"(?P[a-z][^;,|:()\[\]]*?)\s+(?:was\s+|were\s+)?(?:dismissed|nolle\s+pross?ed|nol\s+pross?ed|" + r"nolled|dropped)(?:\s+(?:with|without)\s+prejudice)?)\s*$"), + # what a bare object may not be: a plea, a verdict, a court, a programme, a sentence term + "bare-object-not-offence": ( + r"^(?:as\s+charged|by\b|at\b|in\b|on\b|to\b|of\b|for\b|plea|verdict|jury|court|bench|trial|judge|" + r"deferred|withheld|adjudicat\w*|pbj|probation|jail|prison|fine|fined|sentence\w*|\d|with(?:out)?\s+" + r"prejudice|w/o\s+prejudice|appeal\w*|not\b|guilty|convicted|acquitted|dismissed|charges?\b|counts?\b|" + r"all\b|remaining|other|case|disposition|finding|nolle|nol\b|pros\b|dropped|pending|entered|" + r"verdict|conviction|convict\w*|no\b|none|n/a|unknown|" + # round 4: the words a bare ' DISMISSED' may carry that name no offence + r"later|subsequently|then|thereafter|eventually|ultimately|complaint|indictment|information|" + r"matter|prosecution|proceedings?|it\b|was|were|balance|rest\b|original|greater|lesser|amended|" + r"state\b|prosecutor|da\b|cps\b|crown|motion|petition|see\b|attached|attachment|per\b|as\b|" + r"costs?\b|fees?\b|fines?\b|warrants?\b|capias|forfeiture|bond\b|restitution)"), + # where an object phrase stops: a spaced dash, a date or number, a place or + # time preposition, the sentence + "scope-phrase-stop": ( + r"\s[-–—]+\s|\b\d+(?:[/.\-]\d+)*\b(?![a-z])|\b(?:in|on|at|under|pursuant|per|after|before|upon|" + r"following|dated?|and\s+sentenced|sentenced|with(?:out)?\s+prejudice|probation|jail|prison|fined?)\b"), + # objects that are not an offence ('dismissed on appeal', 'on the merits') + "scope-not-offence": ( + r"^(?:appeal|appeals|review|motion|merits|grounds?|basis|procedural|technical\w*|reason|" + r"lack|speedy|insanity|condition|conditions|completion|payment|probation|violation|vop)$"), + # a dismissal, and an appeal that would make it an appellate one + "dismissal-word": r"\b(?:dismiss\w*|nolle\w*|nol pros\w*|dropped)\b", + # a conviction naming another charge, count or case (the review of batch 1, round 3) + "conviction-other-case": ( + r"\b(?:on|of|to|in|for|under)\s+(?:the\s+|a\s+|an\s+)?(?:[a-z0-9]+\s+){0,3}?(?:other|another|related|" + r"separate|different|unrelated)\s+(?:charges?|cases?|counts?|causes?|matters?|offen[cs]es?|dockets?|" + r"indictments?)\b"), + # a judgment before a trial setting: the setting is not the case's end (round 3) + "judgment-before": ( + r"\b(?:convicted|found guilty|adjudicated guilty|adjudged guilty|guilty verdict|verdict of guilty|" + r"sentenced)\b"), + # a violation or revocation proceeding set for trial (round 3) + "violation-proceeding": ( + r"\b(?:violations?|vop|vocs|revocation|revoke|mtr|mta|motion\s+to\s+(?:revoke|adjudicate))\b"), + # a sealing or expungement word, for the relief-then-dismissal question (round 3) + "relief-sealing": r"\b(?:expung\w*|expunction|seal(?:ed|ing)?)\b", + "appeal-word": r"\b(?:appeal\w*|appellate)\b", + # a later judgment the record reports after another disposition (the review + # of batch 1, R5): a temporal marker then a conviction in the same segment + # ('SUSPENSION OF INDICTMENT; LATER INDICTED AND CONVICTED'), or a deferral + # then a strict adjudication of guilt ('DEFERRED ADJUDICATION; LICENSE + # REVOKED; ADJUDICATED GUILTY') -- never a conviction on another charge + "later-marker": ( + r"\b(?:later|subsequently|thereafter|afterwards|eventually|then|on retrial|at retrial|" + r"after (?:a )?retrial|upon retrial|re-?tried|re-?convicted|re-?indicted|re-?prosecuted|" + r"re-?filed|re-?charged|again|anew|re-?pled|re-?pleaded|re-?sentenced)\b"), + "later-conviction": ( + r"\b(?:(?:re-?)?convicted|conviction (?:was |is )?(?:recorded|entered)|adjudicated guilty|" + r"adjudged guilty|found guilty|guilty verdict|verdict of guilty)\b"), + # a plea of guilty or a sentence after a later marker or the matter brought back + "later-plea": ( + r"\b(?:(?:pled|pleaded|plead|pleads)\s+guilty|guilty\s+plea|plea\s+of\s+guilty|" + r"sentenced(?:\s+to)?)\b"), + # words that put a later conviction on another matter: another case, cause or + # arrest, a completed programme before it (the review of batch 1, round 2) + "other-matter": ( + r"\b(?:(?:separate|different|unrelated)\s+[a-z]+|(?:another|new|other|subsequent)\s+(?:case|matter|charge|" + r"offen[cs]e|cause|docket|citation|file|arrest|entry|county|state|indictment|information|" + # (round 5: 'NEW CAUSE NUMBER' names a docket of the same refiled matter, not another) + r"complaint|incident|conduct)s?(?!\s*(?:no\b\.?|number\b|nr\b\.?|#))|" + r"(?:case|cause|docket|file|citation|indictment)\s*(?:no\.?|" + r"number|nr\.?|#)|see\s+(?:case|entry|cause|docket)|arrested\s+(?:for|on)|new\s+arrest|" + r"civil|forfeiture|contempt)\b|#\s*\w*\d"), + # the same without the docket-number forms: after a deferral or a revocation a cause + # number names this cause as often as another (the review of batch 1, round 3) + "other-matter-explicit": ( + r"\b(?:(?:separate|different|unrelated)\s+[a-z]+|(?:another|new|other|subsequent|companion)\s+(?:case|matter|" + r"charge|offen[cs]e|cause|docket|citation|file|arrest|entry|county|state|indictment|information|" + # (round 5: 'REFILED UNDER NEW CAUSE NUMBER' names this matter's new docket) + r"complaint|incident|conduct)s?(?!\s*(?:no\b\.?|number\b|nr\b\.?|#))|" + r"see\s+(?:case|entry|cause|docket)|arrested\s+(?:for|on)|new\s+arrest|" + r"civil|forfeiture|contempt)\b"), + # a revocation on another cause or case ('PROBATION REVOKED IN ANOTHER CAUSE', + # '(OTHER CASE)'): not this programme's (round 3) + "revocation-other-matter": ( + r"\b(?:(?:another|other|separate|unrelated|prior|previous|different)\s+(?:case|cause|matter|charge|" + r"docket|indictment)s?|on\s+(?:a\s+)?prior\s+case)\b"), + "programme-completed": ( + r"\b(?:complet\w*|discharged|expired|successful\w*|satisfied|terminated\s+(?:early|" + r"successfully|upon\s+(?:successful\s+)?completion))\b"), + # a disposition the later judgment would supersede (with the non-conviction + # cues): a disposal, a suspension, a withdrawal + # (the review of batch 1, round 3: 'released' only without charge or under investigation + # -- never on bail, bond, recognizance or a citation; a suspension only of the + # indictment, prosecution or proceedings, of the sentence's pronouncement or of its + # imposition (a Missouri SIS, superseded when the sentence is imposed) -- never a + # suspended sentence; a breach is not itself a disposal; and the no-charge phrases are) + "earlier-disposal": ( + r"\b(?:suspension\s+of\s+(?:the\s+)?(?:indictment|prosecution|proceedings|(?:sentence\s+)?pronouncement|" + r"the\s+pronouncement)|suspended\s+(?:indictment|prosecution|proceedings)|" + r"(?:indictment|prosecution|proceedings)\s+(?:was\s+|were\s+)?suspended|" + r"caution\w*|discharge\w*|not recorded|no further action|nfa|" + r"released\s*(?:[-:;,]\s*)?(?:without|w/o|with\s+no|no)\s+(?:charges?|further\s+action)|" + r"released\s+(?:(?:under|pending)\s+(?:further\s+)?investigation|nfa)|" + r"(?:imposition\s+of\s+(?:the\s+)?sentence|sentencing)\s+(?:was\s+)?suspended|" + r"suspended\s+imposition(?:\s+of\s+(?:the\s+)?sentence)?|sis|" + r"no\s+charges?\s+(?:was\s+|were\s+)?(?:filed|laid)|charges?\s+not\s+(?:filed|laid)|" + r"(?:da\s+)?declined\s+to\s+file|no\s+proceedings|no\s+file|no[- ]papered|" + r"withdrawn|stet|declined|dropped|deferral|deferred|diver\w*|agreement|fiscal|penalty|" + r"warning|reprimand|community resolution|non-indictment|sobrese\w*|no ejercicio)\b"), + # procedural words that are no disposal for a later judgment to supersede: a plea + # withdrawn or bargained, a release on bail or bond, a stay pending an evaluation + # (round 3; blanked before the earlier-disposal test) + "procedural-not-disposal": ( + r"\b(?:(?:guilty\s+|not\s+guilty\s+)?plea\s+(?:was\s+)?(?:withdrawn|agreement|bargain|deal|offer\w*)|" + r"(?:cited|booked|arrested|detained)\s+(?:and\s+)?released|" + r"released\s+(?:on|upon)\s+(?:his\s+|her\s+)?(?:bond|bail|(?:own\s+|personal\s+)?recogni[sz]ance|" + r"citation|summons|conditions|probation|parole|pretrial\s+supervision)|" + r"released\s+pending\s+(?!(?:further\s+)?investigation\b)\w+|" + r"(?:stayed|continued|adjourned|abated)\s+pending\s+(?!appeal|diversion|acd|acod|deferr|completion|" + r"dismissal|program|successful|payment)(?:a\s+|the\s+)?(?:competency\s+|psychiatric\s+|" + r"mental\s+)?\w+(?:\s+evaluation)?)\b"), + # a same-matter word: the matter brought back, a breach of the disposal, the + # original charge (the review of batch 1, round 2) + # (round 4: 'INDICTMENT RETURNED', 'TRUE BILL RETURNED', the proceedings reinstated, + # and a docket token between 'case' and 'reinstated' -- 'CASE #18-555 REINSTATED') + "same-matter": ( + r"\b(?:(?:re-?)?indicted|re-?filed|re-?charged|re-?opened|re-?tried|retrial|re-?prosecuted|" + r"re-?presented|re-?laid|restored to (?:the )?calendar|on\s+breach|breach(?:ed|es)?\s+(?:of\s+)?" + r"(?:the\s+|its\s+|his\s+|her\s+)?(?:conditions?|caution|order|terms|agreement|bond|undertaking|" + r"probation|supervision|diversion|good behaviou?r)|(?:conditions?|terms|order)\s+breached|" + r"(?:original|same|this)\s+" + r"(?:charge|offen[cs]e|case|matter|count|indictment)|(?:case|charges?|indictment|prosecution|" + r"proceedings|matter)(?:\s+(?:no\.?|number|nr\.?|#)?\s*#?[a-z0-9\-/]*\d[a-z0-9\-/]*)?" + r"\s+(?:was\s+|were\s+)?(?:re-?instated|resumed)|(?:later|subsequently|then|thereafter)\s+" + r"(?:charged|prosecuted|indicted)|prosecution\s+(?:resumed|reinstated)|" + r"indictment\s+(?:was\s+)?returned|true\s+bill\s+(?:was\s+)?(?:returned|found)|" + # round 5: the matter back on remand, a summons reissued, a stet reopened, a plea or + # conviction entered again ('ON REMAND: PLED GUILTY', 'SUMMONS RE-ISSUED; CONVICTED', + # 'REMOVED FROM STET; GUILTY', 'PLED GUILTY AGAIN') -- never a remand in custody or to + # the juvenile court + r"on\s+remand|(?:after|following|upon)\s+(?:the\s+)?remand|" + r"remanded(?!\s+(?:in|into)\s+custody)(?!\s+(?:back\s+)?to\s+(?:the\s+)?juvenile)|" + r"summons\s+(?:was\s+)?(?:re-?issued|re-?served)|proceedings\s+(?:re-?instituted|re-?commenced)|" + r"removed\s+from\s+(?:the\s+)?stet(?:\s+docket)?|(?:taken|removed)\s+off\s+(?:the\s+)?stet|" + r"stet\s+(?:was\s+)?lifted|re-?pled|re-?pleaded|re-?sentenced|re-?convicted|" + r"(?:pled|pleaded|plead)\s+guilty\s+(?:again|anew)|(?:convicted|sentenced)\s+(?:again|anew))\b"), + # a same-matter word that brings the matter back (re-indicted, refiled, reinstated, + # resumed, an indictment returned): a docket number beside it names this case (round 4) + "same-matter-back": ( + r"\b(?:re-?indicted|re-?filed|re-?charged|re-?opened|re-?prosecuted|re-?presented|re-?laid|" + r"re-?instated|resumed|indictment\s+(?:was\s+)?returned|true\s+bill|restored|" + # round 5: on remand, a summons reissued, a stet reopened, a plea entered again + r"on\s+remand|(?:after|following|upon)\s+(?:the\s+)?remand|remanded|summons\s+(?:was\s+)?" + r"(?:re-?issued|re-?served)|proceedings\s+(?:re-?instituted|re-?commenced)|" + r"removed\s+from\s+(?:the\s+)?stet|(?:taken|removed)\s+off\s+(?:the\s+)?stet|stet\s+(?:was\s+)?lifted|" + r"re-?pled|re-?pleaded|re-?sentenced|re-?convicted|(?:pled|pleaded|plead)\s+guilty\s+(?:again|anew)|" + r"(?:convicted|sentenced)\s+(?:again|anew))\b"), + # a same-matter word that names no offence by itself: a bare 'indicted' or 'later + # charged / prosecuted', read as this matter only when no other offence follows (round 3) + "same-matter-weak": ( + r"^(?:indicted|(?:later|subsequently|then|thereafter)\s+(?:charged|prosecuted|indicted))$"), + # the earlier disposals after which a bare 'indicted' brings the matter back + "indictment-withheld": ( + r"\b(?:suspension\s+of\s+(?:the\s+)?indictment|suspended\s+indictment|no[- ]bill\w*|non-indictment|" + r"not\s+indicted|indictment\s+(?:was\s+)?suspended|no\s+charges?\s+(?:was\s+|were\s+)?(?:filed|laid)|" + r"charges?\s+not\s+(?:filed|laid)|declined\s+to\s+(?:file|prosecute|indict)|released\s+without\s+charge)\b"), + "deferral": ( + r"\b(?:deferred adjudication|adjudication deferred|deferred judg(?:e)?ment|" + r"deferred entry of judg(?:e)?ment|pbj|probation before judg(?:e)?ment|adjudication withheld|" + r"withheld adjudication|withhold(?:ing)? of adjudication|sis|suspended imposition(?: of sentence)?|" + r"plea in abeyance|dej|withheld judg(?:e)?ment|judg(?:e)?ment withheld)\b"), + "strict-adjudication": ( + r"\b(?:adjudicated guilty|adjudged guilty|guilt (?:was )?adjudicated|" + r"adjudication of guilt (?:was )?entered|judg(?:e)?ment of (?:conviction|guilt) (?:was )?entered)\b"), + "other-charge": ( + r"^\s*(?:(?:on|of|for|in|with)\s+(?:a|an|the)?\s*(?:new|another|different|separate|subsequent|" + r"unrelated|later|other|companion)\b|(?:of|on)\s+(?:the\s+|a\s+)?(?:probation\s+)?(?:violation|vop)\b)"), + # a later conviction that names its own object ('LATER CONVICTED OF PERJURY') + # is another charge's unless the object is this matter's (the review of batch 1) + # (round 4: a year or date may sit between the verb and 'of' -- 'CONVICTED 2021 OF + # DRINK DRIVING') + "later-object": ( + r"^\s*(?:(?:on\s+|in\s+)?\d{4}(?:-\d\d-\d\d)?\s+)?(?:of|for|on|upon|in)\s+" + r"(?!(?:about\s+|or\s+about\s+)?\d)\S"), + # the same matter brought back ('LATER RE-INDICTED; CONVICTED') + "later-reinstituted": ( + r"\b(?:(?:re-?)?indicted|re-?filed|re-?charged|re-?opened|re-?tried|re-?prosecuted|" + r"restored to (?:the )?calendar|(?:case|charges?|indictment|prosecution|proceedings|matter)\s+" + r"(?:was\s+|were\s+)?re-?instated|indictment\s+(?:was\s+)?returned|true\s+bill\s+(?:returned|found))\b"), + # the matter brought back after it was dropped ('RELEASED WITHOUT CHARGE; LATER + # CHARGED', 'NOLLE PROSEQUI; LATER RE-INDICTED'): with no outcome after it, a + # question (the review of batch 1) + "reinstituted": ( + r"\b(?:re-?indicted|re-?filed|re-?charged|re-?opened|re-?prosecuted|re-?presented|re-?laid|" + r"restored to (?:the )?calendar|(?:case|charges?|indictment|prosecution|proceedings|matter)\s+" + r"(?:was\s+|were\s+)?(?:re-?instated|resumed)|(?:later|subsequently|thereafter|then|afterwards|" + r"eventually)\s+(?:indicted|charged|prosecuted)|true bill (?:returned|found)|" + r"indictment\s+(?:was\s+)?returned|" + # round 5: a summons reissued, proceedings recommenced, a stet lifted + r"summons\s+(?:was\s+)?(?:re-?issued|re-?served)|proceedings\s+(?:re-?instituted|re-?commenced)|" + r"removed\s+from\s+(?:the\s+)?stet|(?:taken|removed)\s+off\s+(?:the\s+)?stet|stet\s+(?:was\s+)?lifted)\b"), + # a bare 'INDICTED' segment after a withheld indictment or a no-bill: the matter + # brought back (round 4: 'NO BILL; INDICTED' asks how it ended) + "indicted-bare": r"^\s*(?:was\s+|were\s+|later\s+|subsequently\s+|then\s+)?indicted\s*$", + # what keeps a brought-back word from bringing THIS matter back: a negation + # within three words, another person or a non-criminal object, a refused or + # struck refiling, another charge (the review of batch 1, round 2) + # (round 4: 'CANNOT BE REFILED') + "reinstituted-negated": r"\b(?:no|nothing|never|not|cannot|can'?t)\b(?:\s+\S+){0,2}\s*$", + "reinstituted-not": ( + r"\b(?:civil|suit|restitution hearing|forfeiture|denied|refused|struck|barred|quashed|" + r"rejected|expun\w*|seal\w*|clerical|correct\w*|nunc pro tunc|unrelated|separate|different)\b"), + # a refiling that is only possible, or history before the dismissal (round 3) + "reinstituted-modal": ( + r"\b(?:may|might|can|could|would|will|shall|cannot|can'?t|couldn'?t|can\s+not|subject\s+to|" + r"eligible\s+to|free\s+to|right\s+to|leave\s+to|without\s+prejudice\s+to)\s+(?:not\s+)?" + r"(?:be\s+|being\s+|have\s+been\s+)?$"), + "reinstituted-history": r"^\s*from\b", + # the matter brought back by an arrest on the same charge, or a bare reinstatement of the + # dismissed case (round 3) + "reinstituted-extra": ( + r"\b(?:re-?arrested\s+(?:on|for)\s+(?:the\s+)?(?:same|original)\s+(?:charges?|offen[cs]es?|matter|" + r"case)|re-?submitted\s+to\s+(?:the\s+)?grand\s+jury)\b"), + "reinstated-bare": ( + r"^\s*(?:(?:the\s+)?(?:case|charges?|indictment|information|complaint|prosecution|proceedings)\s+)?" + r"(?:was\s+|were\s+)?re-?instated\b"), + "later-charged-other": ( + r"^\s*(?:(?:in|on)\s+[a-z0-9 ]+?\s+)?(?:with|for)\s+(?!(?:the\s+)?(?:same|original|this)\b)"), + # an appeal's own outcome ('APPEAL DISMISSED AS UNTIMELY'): not a count's + "appeal-outcome": ( + r"\bappeal\w*\s*:?\s*(?:was\s+|were\s+)?(?:dismissed|withdrawn|abandoned|denied|refused)\b"), + # (the review of batch 1, round 3: a breach of the disposal, never 'BREACH OF THE PEACE') + "later-same-matter": ( + r"^\s*(?:of|for|on|upon|in)\s+(?:the\s+)?(?:breach(?:\s*$|\s+(?:of\s+)?(?:the\s+|its\s+|his\s+|her\s+)?" + r"(?:conditions?|caution|order|terms|agreement|bond|undertaking|probation|supervision|diversion|" + r"good\s+behaviou?r|community\s+order|suspended\s+sentence|licen[cs]e)\b)|retrial|re-?trial|appeal|trial|absentia|" + r"same|original|charges?|counts?|offen[cs]e|indictment|information|complaint|" + r"all\s+(?:charges|counts)|both\s+counts)\b"), + # a penal order (a Strafbefehl) the record does not show final (the review of + # batch 1, R6: StPO s. 410(3) gives the effect of a final judgment only to an + # order not objected to in time) -- and the words that close that question + "penal-order-word": r"\b(?:strafbefehl\w*|penal order|summary penalty order)\b", + "penal-order-objection": ( + r"\b(?:einspruch\w*|einsprache\w*|objection\w*|objected|contested|appealed|" + r"antrag auf gerichtliche entscheidung)\b"), + "penal-order-open": ( + r"\b(?:nicht rechtskr\w*|not (?:yet )?final|eingestellt|discontinued|freispruch|freigesprochen|" + r"acquitted|hauptverhandlung|main hearing|" + # round 5: 'FINAL: NO', 'RECHTSKRAEFTIG: NEIN', 'UNBEKANNT OB RECHTSKRAEFTIG' + r"final\s*[:=]\s*(?:no|nein|non)|rechtskr\w*\s*[:=]\s*(?:nein|no|non|unbekannt|unknown|offen|unklar)|" + r"(?:unbekannt|unklar)\s+ob\s+rechtskr\w*|" + # the review of batch 1, round 3: finality not yet reached, reinstatement granted + # after a missed objection period (StPO s. 44), a rejection of the objection (s. 411(1), + # s. 412) that the record does not show final + r"rechtskraft\s+(?:ist\s+)?(?:noch\s+)?nicht\s+eingetreten|" + r"wiedereinsetzung\w*\s+(?:\S+\s+){0,4}?gew(?:ae|a)hrt|einspruch\w*\s+(?:\S+\s+){0,3}?verworfen)\b"), + # the order itself set aside (not a driving ban in it: 'FAHRVERBOT AUFGEHOBEN') + "penal-order-set-aside": ( + r"\b(?:strafbefehl\w*|penal order|order)\s+(?:(?:wurde|ist|was|is|has been)\s+)?" + r"(?:aufgehoben|annulled|set aside|vacated)\b"), + "penal-order-closed": ( + r"\b(?:kein(?:en|e)?\s+einspruch\w*|ohne\s+einspruch|no\s+objections?(?:\s+(?:was\s+|were\s+)?" + r"(?:lodged|filed|made|raised))?|(?:not|never)\s+objected(?:\s+to)?|" + r"einspruch\w*\s+(?:\S+\s+){0,3}?(?:nicht\s+(?:rechtzeitig\s+)?(?:eingelegt|erhoben)|" + r"zur(?:u|ue)ckgenommen)|(?:not|never)\s+(?:appealed|contested)|nicht\s+angefochten|" + r"einspruchsfrist\s+(?:ist\s+)?(?:abgelaufen|verstrichen)|objection\s+period\s+(?:has\s+)?" + r"(?:expired|lapsed)|keine\s+hauptverhandlung|" + r"objections?\s+(?:was\s+|were\s+)?(?:withdrawn|not\s+(?:filed|lodged|made)(?:\s+in\s+time)?))\b"), + # the record says the order is final ('RECHTSKRAEFTIG', 'BECAME FINAL'): no open + # objection question + # (the review of batch 1, round 3: never a finality word negated or questioned after it -- + # 'RECHTSKRAFT NOCH NICHT EINGETRETEN', 'RECHTSKRAEFTIG? UNBEKANNT') + "penal-order-final": ( + # (round 5: 'RECHTSKRAEFTIG: NEIN', 'UNBEKANNT OB RECHTSKRAEFTIG' show no finality either) + r"\b(?:(? pos and re.sub(r"[^a-z]+", "", _CUE["revocation-filler"].sub(" ", t[max(a, pos):b])) == \ + "adjudicated": + cands.append((re.compile(r"\badjudicated\b").search(t, max(a, pos), b), "guilt")) + if strong: + rev_end = _seg_of(segs, max(pos - 1, 0))[1] + cands += [(m, "guilt") for m in _CUE["adjudication-after-strong"].finditer(t, pos)] + cands += [(m, "plea") for m in _CUE["judgment-after-strong"].finditer(t, pos) + if m.start() >= rev_end] + cands += [(m, "acquittal") for m in _CUE["later-acquittal"].finditer(t, pos) + if m.start() >= rev_end] + for a, b in segs: + if b <= pos: + continue + s = t[max(a, pos):b] + bare = re.sub(r"[^a-z]+", "", _CUE["revocation-filler"].sub(" ", s)) + if bare == "guilty": + cands.append((_BARE_GUILTY.search(t, max(a, pos), b), "guilt")) + elif bare == "sentenced" and a >= rev_end: + # the review of batch 1, round 3: 'DEFERRED ADJUDICATION; REVOKED; SENTENCED' + cands.append((re.compile(r"\bsentenced\b").search(t, max(a, pos), b), "plea")) + elif bare == "notguilty" and a >= rev_end: + # round 4: 'PRETRIAL DIVERSION TERMINATED; JURY TRIAL; NOT GUILTY' -- a bare + # not-guilty segment after the programme ended is the verdict (a plea is + # already removed) + cands.append((re.compile(r"\bnot guilty\b").search(t, max(a, pos), b), "acquittal")) + elif a >= rev_end and _CUE["term-bare"].match(s): + # round 5: 'PBJ REVOKED; 30 DAYS' -- a bare term after the programme itself was + # revoked is the sentence imposed (record_order reads it as 'sentenced') + cands.append((re.compile(r"\d").search(t, max(a, pos), b), "term")) + for m, kind in sorted(cands, key=lambda x: x[0].start()): + if not _adjudication_ok(t, m, segs, strong): + continue + if kind == "plea" and _CUE["deferral"].search(t, pos): + continue # deferred again after the revocation: no judgment read + return m, ("acquittal" if kind == "acquittal" else "guilt") + return None + + +def _revoked_then_adjudicated(t): + """(offset, kind) the record's later judgment reads from when a programme was + revoked (or violated) and a judgment then followed, or None; kind 'guilt' or + 'acquittal'. The revocation must share a segment with a programme word -- or + be a bare 'REVOKED' or 'VIOLATION' segment right after one, or a granted + motion to revoke or adjudicate after one -- and no reinstated / modified / + continued / extended word may come between it and the judgment (or, for a + bare 'convicted' after a mere violation, anywhere after it). A negated + adjudication ('NOT ADJUDICATED GUILTY') is none (the review of batch 1, R2); a + finding of guilt, a plea or a sentence counts only after the programme itself + was revoked, struck or terminated (T1.6(2) gaps); a termination on completion + and a violation dismissed or not found are no revocation (round 2).""" + segs = _segments(t) + for k, (a, b) in enumerate(segs): + s = t[a:b] + granted = _CUE["programme-motion-granted"].search(s) + violation = _CUE["programme-violation"].search(s) + rev = violation or _CUE["revocation"].search(s) or granted + bare_violation = False + if not rev and k > 0 and _CUE["bare-violation"].search(s): + rev, bare_violation = _CUE["bare-violation"].search(s), True + if not rev or (not granted and _CUE["refusal"].search(s)): + continue # no revocation, or a revocation refused + if _CUE["revocation-other-matter"].search(s): + continue # revoked in another cause or case (the review of batch 1, round 3) + if not granted: + strong_words = _CUE["revocation-strong"].findall(s) + if strong_words and all(w in ("terminated", "termination") for w in strong_words): + if _CUE["revocation-not"].search(s) and re.search( + r"\b(?:success\w*|complet\w*|early|satisfactor\w*|expir\w*|discharg\w*)\b", s): + continue # terminated on completion: the programme ended well + elif not strong_words and re.search( + r"\b(?:dismissed|withdrawn|not guilty|not found|unfounded|not proven|not sustained)\b", s): + continue # a violation dismissed or not found + if granted: + rev = granted + bound = bool(_CUE["programme"].search(t[:a + granted.start()])) + strong = True + if bound and re.search(r"\badjudicat|\bmta\b|\bmtag\b|proceed", granted.group(0)) and not ( + _CUE["continuation"].search(t, a + granted.end())): + # round 5: a granted motion to adjudicate guilt is itself the adjudication (Tex. + # Code Crim. Proc. art. 42A.108: the deferral ends with it) -- the record reads + # from the motion, which record_order rewrites to 'adjudicated guilty', whatever + # term or nothing follows ('DEF ADJ; MTA GRANTED; 3 YRS TDCJ') + return a + granted.start(), "mta" + else: + bound = (not bare_violation) and bool( + _CUE["programme-violation"].search(s) or _CUE["programme"].search(s)) + if not bound and k > 0: + bare = _CUE["revocation-filler"].sub(" ", _CUE["bare-violation"].sub( + " ", _CUE["revocation"].sub(" ", s))) + # round 4: a term segment between the programme and its revocation is + # skipped ('DEFERRED; 3 YRS; ACCELERATED; CONVICTED') + j = k - 1 + while j > 0 and _TERM_ONLY.match(t[segs[j][0]:segs[j][1]]): + j -= 1 + prev = t[segs[j][0]:segs[j][1]] + bound = (not re.sub(r"[^a-z0-9]+", "", bare) + and bool(_CUE["programme"].search(prev))) + strong = bool(_CUE["revocation-strong"].search(s)) + if not bound: + continue + after = a + rev.end() + hit = _adjudication_after(t, after, segs, strong) + if not hit: + continue + adj, kind = hit + if _CUE["continuation"].search(t, after, adj.start()): + continue + if (adj.group(0).endswith("convicted") and not strong + and _CUE["continuation"].search(t, adj.end())): + continue # a conviction, then the programme continued: another charge's + seg_start = _seg_of(segs, adj.start())[0] + return (adj.start() if seg_start <= after else seg_start), kind + return None + + +_PRIOR_LABEL = re.compile(r"^\s*(?:prior|previous|previously|earlier)(?:\s+(?:record|history|criminal history|" + r"convictions?|offen[cs]es?))?\s*$") +# a segment that is only a term ('3 YRS', '2 YEARS PROBATION'): round 4 +_TERM_ONLY = re.compile(r"^\s*\d[\d/.\-]*\s*(?:yrs?|years?|mos?|months?|days?|hrs?|hours?)?\s*" + r"(?:probation|supervision|community supervision|deferred)?\s*$") + + +def _weak_back(t_before, seg, hit): + """How a same-matter hit that names no offence by itself reads (the review of batch 1, + round 3): 'other' when an offence follows it ('LATER CHARGED WITH DWI', round 4: + 'INDICTED ON UNRELATED CHARGE'), None when it is a bare 'INDICTED' with no withheld + indictment before it (it may be another matter's), else 'same'.""" + words = re.sub(r"\s+", " ", hit.group(0)).strip() + if not _CUE["same-matter-weak"].match(words): + return "same" + if _CUE["later-charged-other"].match(seg[hit.end():]) or _CUE["other-charge"].match(seg[hit.end():]): + return "other" + if words == "indicted" and not _CUE["indictment-withheld"].search(t_before): + return None + return "same" + + +def _later_judgment(t): + """(offset, kind, others) for a judgment the record reports after another disposition + (the review of batch 1, R5 and round 2), or None. kind 'read': the record + reads from `offset` -- the matter was brought back (re-indicted, refiled, + retried, a breached disposal, 'later charged') and then convicted, pleaded, + sentenced or acquitted, or a deferral was followed by a strict adjudication + of guilt. kind 'question': a temporal marker then a conviction or a plea with + nothing saying whether it is this matter's (CF-15 option B: 'is the later + conviction on this charge or on another matter?'). Nothing is read when the + judgment is another person's, earlier in time, refused or asked for, on + another charge, case, cause or arrest, names its own offence (CF-15: 'LATER + CONVICTED OF PERJURY'), or follows a completed programme -- and those + segments are `others`, (start, end) spans the caller removes, so another + matter's conviction words are not read as this record's (round 2). + + Round 3: a withdrawn or bargained plea, a release on bail, bond or a citation and a + stay pending an evaluation are no disposal to supersede; a bare 'INDICTED' brings the + matter back only after a withheld indictment, and 'LATER CHARGED WITH ' names + another matter; a strict adjudication after a deferral is that deferral's judgment + whatever object, date or cause number follows it (unless a temporal marker puts an + offence of its own after it: CF-15); a later sentence to pay costs, or to the + deferral's own supervision, is no judgment of guilt.""" + segs = _segments(t) + question, others = None, [] + for k, (a, b) in enumerate(segs): + if k == 0: + continue + before = t[:a] + before_d = _CUE["procedural-not-disposal"].sub(_blank, before) + if not (_NON_CONVICTION_CUES.search(_fold(before_d)) or _CUE["earlier-disposal"].search(before_d) + or _CUE["procedural-not-disposal"].search(before)): + continue # nothing earlier for a later judgment to supersede -- round 4: a + # procedural step alone (a release on bond) enters, so a later conviction on + # another matter after it is removed, never read as this record's + s = t[a:b] + prev = t[segs[k - 1][0]:segs[k - 1][1]] + nxt = t[segs[k + 1][0]:segs[k + 1][1]] if k + 1 < len(segs) else "" + if (_CUE["other-person"].search(s) or _CUE["earlier-in-time"].search(s) + or _PRIOR_LABEL.match(prev)): + continue + back_s = _CUE["same-matter"].search(s) + if back_s and _CUE["other-person-family"].search(s[:back_s.start()]): + continue # round 5: "DEFENDANT'S BROTHER INDICTED" brings nothing back + # round 4: a bare 'REINSTATED' segment after a dismissal is the matter brought back + back_p = _CUE["same-matter"].search(prev) or ( + _CUE["reinstated-bare"].match(prev) + if _CUE["dismissal-word"].search(t[:segs[k - 1][0]]) else None) + if back_p and _CUE["other-person-family"].search(prev[:back_p.start()]): + back_p = None # round 5: "DEFENDANT'S BROTHER INDICTED; CONVICTED" is the brother's + weak_s = _weak_back(before, s, back_s) if back_s else None + weak_p = _weak_back(t[:segs[k - 1][0]], prev, back_p) if back_p else None + unsure = bool((back_s and weak_s is None) or (back_p and weak_p is None)) + if back_s and weak_s != "same": + back_s = None + charged_other_p = weak_p == "other" + # round 4: beside a word that brings the matter back (refiled, re-indicted, + # reinstated) a docket number names this case -- only the explicit other-matter + # words put it elsewhere ('CASE #18-555 REINSTATED 2019; PLED GUILTY') + other_cue = ("other-matter-explicit" if back_p and _CUE["same-matter-back"].search(back_p.group(0)) + else "other-matter") + if back_p and (weak_p != "same" or _CUE["other-charge"].search(prev[back_p.end():]) + or _CUE[other_cue].search(prev)): + back_p = None # 'LATER RE-INDICTED ON UNRELATED CHARGE', 'LATER CHARGED WITH DWI' + # an arrest or a pending charge before it: the same matter ended in the judgment + open_before = bool(re.search(r"\b(?:pending|awaiting|arrested|charged|open case|warrant)\b", before) + and not _NON_CONVICTION_CUES.search(_fold(re.sub( + r"\b(?:pending|awaiting trial|awaiting)\b", " ", before_d)))) + marker = _CUE["later-marker"].search(s) + cands = [(m, "guilt") for m in _CUE["later-conviction"].finditer(s)] + cands += [(m, "plea") for m in _CUE["later-plea"].finditer(s)] + cands += [(m, "acquittal") for m in _CUE["later-acquittal"].finditer(s)] + if (back_s or back_p) and re.sub(r"[^a-z]+", "", _CUE["revocation-filler"].sub(" ", s)) == "guilty": + # round 5: a bare 'GUILTY' after the matter was brought back ('STET; REOPENED; + # GUILTY', 'REMOVED FROM STET; GUILTY') is the later judgment + cands.append((_BARE_GUILTY.search(s), "guilt")) + if _CUE["deferral"].search(before) and not _deferral_count_split(t): + cands += [(m, "strict") for m in _CUE["strict-adjudication"].finditer(s)] + procedural_only = not (_NON_CONVICTION_CUES.search(_fold(before_d)) + or _CUE["earlier-disposal"].search(before_d)) + # a strict adjudication reads before the same words read as a conviction + for m, kind in sorted(cands, key=lambda x: (x[0].start(), x[1] != "strict")): + rest = s[m.end():] + if _negated_at(s, m.start()): + continue + if _CUE["other-charge"].search(rest): + if procedural_only and kind in ("guilt", "plea") and marker and marker.start() <= m.start(): + others.append((a, b)) # round 4: 'LATER CONVICTED OF UNRELATED BURGLARY' + break + continue + if procedural_only: + if kind in ("guilt", "plea") and marker and marker.start() <= m.start(): + # a stay, a release or a withdrawn plea, then the judgment -- round 4: + # never one on another matter ('LATER CONVICTED OF DWI IN ANOTHER CASE', + # '(SEPARATE CASE)', 'FOR CONTEMPT'), which is removed; one naming an + # offence of its own is this matter's only where the earlier words name + # that offence too ('ARRESTED FOR BURGLARY; ...; LATER CONVICTED OF + # BURGLARY'), else the later-conviction question (CF-15 option B) + if _CUE["other-matter"].search(s) or _CUE["other-matter"].search(nxt): + others.append((a, b)) + break + # round 5: a bare procedural step is no disposal -- the matter is still + # open, and the conviction after it closes it whether or not it names its + # offence ('RELEASED ON BOND; LATER CONVICTED OF ASSAULT' is this matter's, + # as 'ARRESTED; RELEASED ON BAIL; LATER CONVICTED OF BURGLARY' was); round 4 + # had removed the named form as another matter's, leaving nothing + return a, "read", [] + continue + if kind == "plea" and (re.match(r"^\s*(?:to\s+)?pay\s+(?:(?:the\s+)?court\s+)?(?:costs|fees)\b", rest) or ( + _CUE["deferral"].search(before) and re.match( + r"^\s*(?:to\s+)?(?:\d+\s*\w+\s+(?:of\s+)?)?(?:deferred\s+\w+\s+)?(?:community supervision|" + r"probation|supervision)\b", rest))): + continue # sentenced to pay costs, or to the deferral's own supervision + back = back_s or back_p + eligible = kind == "strict" or back or (marker and marker.start() <= m.start()) + if charged_other_p or weak_s == "other": + if eligible or charged_other_p: + others.append((segs[k - 1][0] if charged_other_p else a, b)) + break # charged with another offence, then convicted + if kind == "strict": + other_here = _CUE["other-matter-explicit"].search(s) + else: + other_here = (_CUE[other_cue].search(s) or _CUE["other-matter"].search(nxt) + or (back_p and _CUE[other_cue].search(prev))) + if other_here: + if eligible and kind != "acquittal": + nb = (segs[k + 1][1] if (kind != "strict" and k + 1 < len(segs) + and not _CUE["other-matter"].search(s)) else b) + others.append((a, nb)) + break # another case, cause, arrest or offence + if unsure and kind in ("guilt", "plea") and not back and question is None: + question = a # a bare 'INDICTED' before it: this matter's or another's? + break + named = bool(not back and _CUE["later-object"].search(rest) + and not _CUE["later-same-matter"].search(rest)) + if kind == "strict": + # a strict adjudication names this deferral's matter whatever follows it, + # unless a temporal marker puts an offence of its own after it (CF-15) + named = named and bool(marker and marker.start() <= m.start() + and re.match(r"^\s*of\s", rest)) + if kind in ("guilt", "strict") and named: + # round 4: after a caution, a discharge, a deferral or another disposal + # that is no dismissal, a conviction naming its own offence is another + # charge's even with no temporal marker ('CAUTION; ...; CONVICTED OF + # FTA'); beside a dismissal or a count the split-counts rule reads it + if eligible or (kind == "guilt" and _CUE["earlier-disposal"].search(before_d) + and not _CUE["outcome-word"].search(before_d) + and not _CUE["count-scope"].search(t)): + others.append((a, b)) + break # names its own offence: another charge's (CF-15) + if not back and _CUE["programme-completed"].search(before): + if eligible and kind != "acquittal": + others.append((a, b)) + break # after a completed programme: another matter + if kind == "strict": + if _CUE["refusal"].search(s) or _CUE["new-trial-asked"].search(s[:m.start()]) or re.search( + r"\b(?:petition|motion|request)s?\s+(?:to\s+have|that)\b", s[:m.start()]) \ + or _CUE["hearing-notice"].search(s): + continue # (round 4: a hearing notice names the adjudication, makes none) + return a, "read", [] + if back or (open_before and marker): + # (round 4: from the back word itself when it shares the segment with the + # earlier outcome -- 'CHARGE WITHDRAWN AND RECHARGED; CONVICTED') + start = (segs[k - 1][0] + back_p.start() if (back_p and not back_s) + else a + min(back_s.start(), m.start()) if back_s else a) + bp = back_p if (back_p and not back_s) else back_s + if bp and re.search(r"\bstet\b", bp.group(0)): + # round 5: 'REMOVED FROM STET; GUILTY' reads after the stet phrase, which + # names the earlier disposal + start = (segs[k - 1][0] if (back_p and not back_s) else a) + bp.end() + return start, "read", [] + if marker and marker.start() <= m.start() and kind != "acquittal" and question is None: + question = a + if question is not None: + return question, "question", others + return (None, None, others) if others else None + + +def _deferral_count_split(t): + """True when a deferral names one count and an adjudication or finding of + guilt another ('DEFERRED ADJUDICATION ON COUNT 1; COUNT 2 ADJUDICATED GUILTY': + the review of batch 1, round 2, M4).""" + deferred, judged = set(), set() + # (round 5: a count label joined by a colon -- 'COUNT 1: GUILTY' -- stays one segment) + for a, b in _segments(t, _MAJOR_BREAK): + s = t[a:b] + scope = _CUE["count-scope"].search(s) + if not scope: + continue + key = re.sub(r"[^a-z0-9]+", " ", scope.group(0)).strip() + # round 5: a bare 'GUILTY' or a bare 'DEFERRED' beside the count label ('CT1 GUILTY; + # CT2 DEF ADJ', 'COUNT 1: GUILTY; COUNT 2: DEFERRED') + bare = re.sub(r"[^a-z]+", "", _CUE["revocation-filler"].sub(" ", _CUE["count-scope"].sub(" ", s))) + if _CUE["deferral"].search(s) or bare == "deferred": + deferred.add(key) + elif _CUE["strict-adjudication"].search(s) or re.search( + r"\b(?:found guilty|convicted|guilty verdict|verdict of guilty)\b", s) or bare == "guilty": + judged.add(key) + return bool(deferred and judged and deferred != judged) + + +def deferral_count_split(disposition): + """_deferral_count_split on the record's words (the server lets split-counts + answer for a deferral entry then).""" + return _deferral_count_split(_accent_fold(expand_disposition(disposition or ""))) + + +def _adult_court(t, flags): + """t with the adult-court rule applied (T1.6(3); the review of batch 1, R3 and + round 2). A transfer that took effect strips the juvenile words and raises + 'adult-court' -- unless the record later returns the case to juvenile court or + reports a delinquency adjudication, when the record reads from that later + judgment. A transfer that did not take effect (negated, refused, withdrawn, + reversed, only a hearing or motion pending, another person's, or for another + offence) is cut with its motion and refusal words, and the juvenile words + stay. + + Round 3: for a transfer that took effect only the adult qualifier is cut -- the verb + of judgment stays ('CONVICTED AS AN ADULT' reads 'convicted'); a transfer in a prior + or another case, a family member's, a future or hypothetical one ('WILL BE TRIED AS AN + ADULT', 'COULD HAVE BEEN PROSECUTED AS AN ADULT') or one answered 'NO' did not take + effect for this matter; and a discretionary transfer to a district court counts + after juvenile words (Texas).""" + t = _CUE["adult-certification-other"].sub(" ; ", t) + segs = _segments(t) + effective, cuts = [], [] + found = list(_CUE["adult-court"].finditer(t)) + found += [m for m in _CUE["adult-district-transfer"].finditer(t) + if _CUE["juvenile-words"].search(t, 0, m.start()) + and not any(x.start() <= m.start() < x.end() for x in found)] + for m in sorted(found, key=lambda x: x.start()): + a, b = _seg_of(segs, m.start()) + seg = t[a:b] + answer_no = _CUE["adult-answer-no"].match(t[m.end():]) + # round 4: 'LATER CHARGED AS AN ADULT FOR ROBBERY' is a later, separate charge -- + # not this matter's transfer; the segment goes, the juvenile words stay + later_other = bool(re.search(r"\b(?:later|subsequently|then|thereafter|afterwards)\s+$", + t[max(a, m.start() - 30):m.start()]) + and _CUE["later-charged-other"].match(t[m.end():b])) + # round 5: a charge, transfer or conviction as an adult 'IN A NEW CASE' / '(NEW CASE)' + # is another matter's: the segment goes, the juvenile words stay + if later_other or _CUE["adult-other-case"].search(t[m.end():b]): + cuts.append((a, b)) + continue + # round 5: 'SENTENCED AS AN ADULT' after a delinquency adjudication, with no transfer or + # trial named, is a blended or determinate sentence on the juvenile adjudication -- only + # the qualifier goes, the juvenile words stay (a HELD characterisation: CF-21) + if (m.group(0).startswith("sentenced") and _CUE["delinquency-outcome"].search(t, 0, m.start()) + and not any(x is not m and not x.group(0).startswith("sentenced") for x in found)): + q = _CUE["adult-qualifier"].match(m.group(0)) + cuts.append((m.start() + q.start(1), m.end()) if q else (m.start(), m.end())) + continue + taken = not (_negated_at(t, m.start()) or _CUE["adult-not-effective"].search(seg) + or _CUE["other-person"].search(seg) or _CUE["other-person-family"].search(seg) + or _CUE["earlier-in-time"].search(seg) or _CUE["adult-other-matter"].search(seg) + or _CUE["adult-modal"].search(t[max(a, m.start() - 40):m.start()]) + or re.search(r"\b(?:future|hypothetical\w*|eligible|possible|possibly)\b", seg) + or answer_no + or (_CUE["adult-procedural"].search(seg) + and not re.search(r"\b(?:granted|ordered)\b", seg))) + if taken: + effective.append(m) + continue + lead = _CUE["adult-lead"].search(t[a:m.start()]) + start = a + lead.start() if lead else m.start() + tail = _CUE["adult-tail"].match(t[m.end():b]) + end = m.end() + (tail.end() if tail else 0) + if answer_no: + end = max(end, m.end() + answer_no.end()) + cuts.append((start, end)) + if effective: + last = max(m.end() for m in effective) + ret = _CUE["juvenile-return"].search(t, last) + dq = next((d for d in _CUE["delinquency-outcome"].finditer(t, last) + if not _CUE["earlier-in-time"].search(t[slice(*_seg_of(segs, d.start()))]) + # a conviction in adult court before it: not the case sent back + and not _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", t[last:d.start()]))), None) + if ret or dq: + # the case went back to juvenile court: the later judgment is read + if ret and (not dq or ret.start() <= dq.start()): + return " ; " + t[ret.end():] + return t[_seg_of(segs, dq.start())[0]:] + for m in reversed(effective): + q = _CUE["adult-qualifier"].match(m.group(0)) + # the verb of judgment stays; only 'as an adult' / 'in adult court' is cut + cuts.append((m.start() + q.start(1), m.end()) if q else (m.start(), m.end())) + flags.add("adult-court") + for start, end in sorted(cuts, reverse=True): + t = t[:start] + " ; " + t[end:] + if effective: + t = _CUE["juvenile-words"].sub(" ", t) + return t + + +def _refused_spans(t): + """The spans (start, end) of relief, programme and offer words a refusal binds + to, with their motion and refusal words (T1.6(4); the review of batch 1, R4 and + round 2). The refusal binds only when it follows the word with nothing but + filler between ('EXPUNGEMENT OF ARREST RECORD DENIED', 'CAUTION OFFERED AND + DECLINED'), when the word is its object ('DECLINED CAUTION'), when it is a + segment of its own right after the word's ('OFFERED CONDITIONAL CAUTION; + REFUSED'), or -- for an offer -- when the person elected a court hearing + instead. A refusal of a test, advice, a charge or a prosecution binds to + nothing ('PBJ FOR DUI (REFUSED BREATH TEST)', 'CAUTION ADMINISTERED AFTER CPS + DECLINED TO CHARGE'); a denied motion to revoke keeps the programme. + + Round 3: a refused motion AGAINST relief ('STATE'S MOTION TO SET ASIDE EXPUNGEMENT + DENIED') is cut from the motion, so the relief stands and neither relief word reads + as granted; a relief petition dismissed or withdrawn grants nothing; the conditions + of an offer refused are a refusal of the offer; and an offer not completed, not paid, + failed, breached or appealed grants nothing when conviction words follow it in a + later segment ('FISCAL FINE; NOT PAID; PROSECUTED; CONVICTED').""" + segs = _segments(t, _REFUSAL_BREAK) + spans = [] + + def words(a, b): + s = t[a:b] + found = [] + for cue in ("relief-word", "programme", "offer-word"): + for m in _CUE[cue].finditer(s): + if cue == "programme" and _CUE["programme-motion"].search(s): + continue # a motion against the programme: its denial keeps it + found.append((a + m.start(), a + m.end(), cue)) + return found + + def lead(a, start): + against = _CUE["refusal-motion-against"].search(t[a:start]) + if against: + return a + against.start() + ml = _CUE["refusal-motion-lead"].search(t[a:start]) + return a + ml.start() if ml else start + + def convicted_after(pos): + # conviction words after the offer, on no other matter and naming no offence of + # their own ('UNPAID; CONVICTED OF FAILURE TO PAY' is another offence's) + rest = _NEGATED_CONVICTION.sub(" ", _fold(t[pos:])) + return bool(any(not re.match(r"^\s*(?:of|for)\s+(?!(?:the\s+)?(?:same|original)\b)\S", rest[m.end():]) + for m in _CONVICTION_CUES.finditer(rest)) + and not _CUE["other-matter-explicit"].search(t[pos:])) + + def span(a, x, end): + # round 4: what the cut keeps -- the relief word of a refused motion AGAINST it + # ("PROSECUTION'S APPLICATION TO REVOKE PARDON REFUSED" keeps 'pardon'), and + # 'conviction' for a refused motion to set aside a conviction, verdict, judgment + # or plea ('MOTION TO SET ASIDE CONVICTION DENIED': what the motion attacked stands) + start = lead(a, x[0]) + keep = "" + if start < x[0] and _CUE["refusal-motion-against"].search(t[a:x[0]]): + keep = t[x[0]:x[1]] + elif x[2] == "relief-word" and _CUE["refusal-object-stands"].match(t[x[1]:end]): + # (round 5: a refused motion to set aside a dismissal, an acquittal or relief keeps + # that outcome's own word -- 'MOTION TO SET ASIDE DISMISSAL DENIED' is the dismissal) + obj = re.search(r"[a-z]+$", _CUE["refusal-object-stands"].match(t[x[1]:end]).group(0)) + keep = _REFUSAL_OBJECT_KEPT.get(obj.group(0) if obj else "", "conviction") + return (start, end, keep) + + for m in _CUE["relief-petition-dismissed"].finditer(t): + a = _seg_of(segs, m.start())[0] + spans.append((lead(a, m.start()), m.end(), "")) + for k, (a, b) in enumerate(segs): + xs = words(a, b) + for r in _CUE["refusal"].finditer(t, a, b): + if _CUE["refusal-object"].match(t[r.end():b]): + continue # 'REFUSED BREATH TEST', 'DECLINED TO CHARGE' + before = [x for x in xs if x[1] <= r.start() + and _CUE["refusal-between"].match(t[x[1]:r.start()])] + if before: + x = max(before, key=lambda x: x[1]) + by = _CUE["refusal-by"].match(t[r.end():b]) + spans.append(span(a, x, r.end() + (by.end() if by else 0))) + continue + head = _CUE["refusal-lead"].match(t[r.end():b]) + after = [x for x in xs if x[0] == r.end() + head.end()] + if after: + spans.append((r.start(), after[0][1], "")) + offers = [x for x in xs if x[2] == "offer-word"] + if offers and convicted_after(b): + for r in _CUE["non-completion"].finditer(t, a, b): + before = [x for x in offers if x[1] <= r.start() + and _CUE["refusal-between"].match(t[x[1]:r.start()])] + if before: + spans.append((lead(a, max(before, key=lambda x: x[1])[0]), r.end(), "")) + if k + 1 < len(segs) and xs: + na, nb = segs[k + 1] + nxt = t[na:nb] + if (offers and _CUE["non-completion-bare"].match(nxt) and convicted_after(nb)): + spans.append((lead(a, offers[-1][0]), nb, "")) + continue + if _CUE["grant-word"].search(t[a:b]) and not _CUE["elected-court"].match(nxt): + continue + if _CUE["refusal-bare"].match(nxt): + x = xs[-1] + spans.append(span(a, x, nb)) + elif _CUE["elected-court"].match(nxt) and any(c == "offer-word" for *_, c in xs): + spans.append((a, nb, "")) + return spans + + +_MAJOR_BREAK = re.compile(r"\s*[;,|]\s*") + + +def _split_counts(t, folded, live, conv, nodisp): + """The split-counts question (T1.6(5), the review of batch 1): conviction on + some counts or charges and another outcome on others -- by count words (an + appeal's outcome is not a count's), a plea to a count, an unreported count, or + outcomes that name different offences, compared as whole phrases ('GUILTY OF + POSSESSION; NOT GUILTY OF POSSESSION WITH INTENT') -- also with no preposition, + an outcome and an offence alone between ';', ',' or '|' ('GUILTY - ASSAULT; NOT + GUILTY - ROBBERY', 'CONVICTED ASSAULT, ACQUITTED ROBBERY': round 3).""" + conv_any = conv or _CUE["plea-to-count"].search(folded) + if not conv_any: + return False + no_appeal = _CUE["appeal-outcome"].sub(" ", strip_appeal_upheld(folded)) + scoped = _CUE["count-scope"].search(folded) + if scoped and (_CUE["outcome-word"].search(no_appeal) or nodisp): + return True + + def phrases(text, cue): + out = set() + for m in _CUE[cue].finditer(text): + first = re.match(r"[a-z][a-z0-9]*", m.group(1)).group(0) + if _CUE["scope-not-offence"].search(first): + continue + # round 4: a numbered specification, count or charge keeps its number + # ('GUILTY OF SPECIFICATION 1; NOT GUILTY OF SPECIFICATION 2' on a military record) + numbered = re.match(r"(specifications?|counts?|charges?|cts?)\s*#?\s*(\d+|[ivx]+)\b", m.group(1)) + if numbered: + out.add(f"{numbered.group(1)} {numbered.group(2)}") + continue + head = _CUE["scope-phrase-stop"].split(m.group(1), maxsplit=1)[0] + ws = re.findall(r"[a-z0-9]+", head)[:5] + if ws: + out.add(" ".join(ws)) + return out + + def bare(text, cue): + out = set() + segs = _segments(text, _MAJOR_BREAK) + for i, (a, b) in enumerate(segs): + if cue == "bare-conviction-offence" and re.match( + r"^\s*(?:guilty|convicted)\s*[-:–—]\s*(?:\$|\d|time served|fine|jail|probation)", + text[a:b]): + # round 5: a labelled conviction whose label is a term ('GUILTY - 30 DAYS') + # is a win beside a labelled loss ('NOT GUILTY - ROBBERY') + out.add("(term)") + continue + if cue == "bare-outcome-offence" and re.fullmatch( + r"\s*(?:dismissed|nolle\w*|nol\s+pros\w*|dropped)\s*", text[a:b]) and any( + _CUE["scoped-conviction-phrase"].search(text[x:y]) for x, y in segs[i + 1:]): + # round 5: a bare dismissal before a conviction naming another offence + # ('DISMISSED; CONVICTED OF A DIFFERENT OFFENSE ARISING FROM SAME INCIDENT') is + # the plea-down split + out.add("(bare) dismissed") + continue + m = _CUE[cue].match(text[a:b]) + if not m: + continue + o = (m.group("o") or m.group("o2") or (m.group("o3") if "o3" in m.groupdict() else None) + or "").strip(" -:") + if not o or _CUE["bare-object-not-offence"].match(o): + continue + if _CUE["scope-not-offence"].search(re.match(r"[a-z]*", o).group(0)): + # round 4: 'DISMISSED - SPEEDY TRIAL' beside 'GUILTY - DUI' is a dismissal + # for a reason -- an outcome of its own, never the offence's name + if cue == "bare-outcome-offence": + out.add("(reason) " + re.match(r"[a-z]*", o).group(0)) + continue + ws = re.findall(r"[a-z0-9]+", o)[:5] + if ws and len(re.findall(r"[a-z0-9]+", o)) <= 5: + out.add(" ".join(ws)) + return out + t_neg = _NEGATED.sub(_blank, t) + t_win = _NEGATED_CONVICTION.sub(_blank, t_neg) + wins = phrases(t_win, "scoped-conviction-phrase") | bare(t_win, "bare-conviction-offence") + losses = phrases(t_neg, "scoped-outcome-phrase") | bare(t_neg, "bare-outcome-offence") + return bool(wins and losses and wins != losses) + + +def _offence_label(text): + """True when a segment is only an offence label ('DWLS', 'OPEN CONTAINER') -- no + outcome or conviction word, no generic, relief, count, date or procedural word (the + review of batch 1, round 3).""" + words = re.findall(r"[a-z0-9§]+", text) + if not words or len(words) > 5 or not any(re.search(r"[a-z]", w) for w in words): + return False + if all(len(w) == 1 for w in words): + return False # round 5: 'N/A' is no offence label + obj = " ".join(words) + return not (_CONVICTION_CUES.search(obj) or _NON_CONVICTION_CUES.search(obj) + or _CUE["outcome-word"].search(obj) or _CUE["dismissal-object-generic"].match(obj) + or _CUE["dismissal-object-relief"].search(obj) or _CUE["count-scope"].search(obj) + or _CUE["scope-not-offence"].search(words[0]) or _CUE["bare-object-not-offence"].match(obj) + or _CUE["dismissal-object-non-charge"].match(obj) + or re.search(r"\b(?:prejudice|disposition|date|court|judge|case|cause|docket|speedy|" + r"merits|state|motion|bond|warrant|hearing|reason|county|parish|district|" + r"municipal|circuit|superior|department|dept|police|sheriff|da|prosecutor|" + r"attorney|division|clerk|" + # round 4: costs, fees, a capias, a forfeiture, a hold, an order, + # 'see attached', a dismissal's reason + r"costs?|fees?|fines?|capias|forfeiture|restitution|counsel|hold|order|" + r"see|attached|attachment|per|as|insufficient|evidence|witness|" + r"unavailable|interest|justice|complainant|victim|w/o)\b", obj)) + + +def _dismissal_after(t, t_live, conv): + """What a dismissal after conviction words asks (the critic's + post-conviction dismissal; the review of batch 1, round 2, M1): 'split' when + the dismissed thing is a charge the record names ('DWLS DISMISSED', 'CHARGE OF + BURGLARY NOLLE PROSSED') -- round 3: also an offence label joined to the + dismissal by a spaced dash or a colon ('DUI - GUILTY; DWLS - DISMISSED', 'GUILTY + - DUI; DISMISSED - DWLS') --, 'question' when it is the charge or the case itself, + or a relief's own dismissal ('was judgment entered before the dismissal?'), + None when nothing is dismissed after them.""" + segs = _segments(t) + kind = None + for m in _CUE["dismissal-word"].finditer(t_live, conv.end()): + i = next((k for k, (x, y) in enumerate(segs) if x <= m.start() < y), None) + a = segs[i][0] if i is not None else m.start() + obj = " ".join(re.findall(r"[a-z0-9§]+", t[max(a, conv.end()):m.start()])) + if not obj and i is not None: + joined = (lambda p, q: re.search(r"[:\-–—]", t[p:q]) is not None) + label = None + if i > 0 and segs[i - 1][0] >= conv.end() and joined(segs[i - 1][1], segs[i][0]): + label = t[segs[i - 1][0]:segs[i - 1][1]] + elif (i + 1 < len(segs) and joined(segs[i][1], segs[i + 1][0]) + and not re.findall(r"[a-z0-9]+", t[m.end():segs[i][1]])): + label = t[segs[i + 1][0]:segs[i + 1][1]] + if label is not None and _offence_label(label): + return "split" + if label is not None and (_CUE["dismissal-object-non-charge"].match( + " ".join(re.findall(r"[a-z0-9/]+", label))) + or _CUE["dismissal-object-other-case"].search(label)): + continue # round 4: 'DISMISSED - COSTS', 'DISMISSED - WARRANT': no charge + if label is None and not re.findall(r"[a-z0-9]+", t[m.end():segs[i][1]]): + # round 5: the label joined to the dismissal was cut by an earlier rule (another + # person's reason: 'DISMISSED - VICTIM UNAVAILABLE') -- a dismissal for a reason + # beside a labelled conviction is the split, as 'DISMISSED - SPEEDY TRIAL' is + if any(_CUE["bare-conviction-offence"].match(t[x:y]) for x, y in _segments(t, _MAJOR_BREAK) + if x < m.start()): + return "split" + if _CUE["dismissal-object-other-case"].search(obj) and not re.search( + r"\b(?:charges?|counts?)\b", obj): + continue # round 4: 'SEPARATE CASE DISMISSED' -- another case's outcome + if (_CUE["dismissal-object-generic"].match(obj) or _CUE["dismissal-object-relief"].search(obj) + or _CUE["count-scope"].search(obj)): + kind = kind or "question" + continue + return "split" + return kind + + +def _penal_order_state(t): + """How a penal order (a Strafbefehl) stands on the record's words (the review + of batch 1, R6 and round 2): None when the record names none or shows it final + ('RECHTSKRAEFTIG', 'EINSPRUCHSFRIST ABGELAUFEN', 'KEINE HAUPTVERHANDLUNG'); 'open' + when it names an objection, a setting-aside of the order, a discontinuance, a + main hearing or an acquittal with nothing after that closes it; ('read', offset) + when an objection is followed by an English acquittal the curated entries read + ('ACQUITTED', 'FOUND NOT GUILTY'): the record reads from there (CF-16 option B for + a stated outcome); 'convicted' when a conviction judgment follows the objection + (no question). + + Round 3: a finality word negated or questioned after it ('RECHTSKRAFT NOCH NICHT + EINGETRETEN', 'RECHTSKRAEFTIG? UNBEKANNT') is no finality; a rejection of the + objection ('EINSPRUCH VERWORFEN', StPO s. 411(1) / s. 412, open to challenge) and + reinstatement after a missed period (s. 44) leave the question open; a German + acquittal or any discontinuance after the objection keeps the question (no curated + entry reads 'Freispruch' or 'eingestellt'); a judgment ('URTEIL') imposing a fine or + imprisonment is the conviction.""" + if not _CUE["penal-order-word"].search(t): + return None + segs = _segments(t) + closed = _CUE["penal-order-closed"].sub(_blank, t) + # another person's outcome ('CO-ACCUSED ACQUITTED') is not the order's; a bare + # objection segment that a later closing phrase naming the objection answers + # ('EINSPRUCH; EINSPRUCH ZURUECKGENOMMEN') is closed too (round 4) + for a, b in segs: + if _CUE["other-person"].search(closed[a:b]) or ( + _CUE["penal-order-objection-bare"].match(closed[a:b]) + and _CUE["penal-order-closed"].search(t, b)): + closed = closed[:a] + " " * (b - a) + closed[b:] + obj = _CUE["penal-order-objection"].search(closed) + if obj: + outs = [(m, "read" if _CUE["penal-order-read"].fullmatch(m.group(0)) else "open") + for m in _CUE["penal-order-acquittal"].finditer(closed, obj.end())] + outs += [(m, "open") for m in _CUE["penal-order-discontinued"].finditer(closed, obj.end())] + # round 4: an English conviction word after the objection is read from (the + # conviction entry answers the generic penal-order entry's record too) + outs += [(m, "read" if _CUE["penal-order-read-conviction"].fullmatch(m.group(0)) else "convicted") + for m in _CUE["penal-order-convicted"].finditer(closed, obj.end())] + outs = [o for o in outs if not _negated_at(closed, o[0].start())] + if outs: + # (round 5: the outcome after the objection is read before any finality word -- + # 'EINSPRUCH; FREISPRUCH; RECHTSKRAEFTIG' is the acquittal, final) + m, kind = min(outs, key=lambda o: o[0].start()) + if kind == "read": + return "read", _seg_of(segs, m.start())[0] + return kind + if _CUE["penal-order-final"].search(closed): + return None # an objection, no outcome stated, the order shown final + return "open" + if _CUE["penal-order-final"].search(t): + return None + if _CUE["penal-order-open"].search(closed) or _CUE["penal-order-set-aside"].search(closed): + return "open" + return None + + +def _penal_order_open(t): + """True when the record names a penal order it does not show final (the + review of batch 1, R6): _penal_order_state 'open'.""" + return _penal_order_state(t) == "open" + + +_RECORD_ORDER_CACHE = {} +_BARE_GUILTY = re.compile(r"\bguilty\b") + + +def _later_outcome(rest, unknown=False): + """True when the words after a procedural step (a trial set, a new trial + ordered, a matter brought back) report an outcome for this person and this + matter (the review of batch 1, R1, R7 and round 2): a disposition phrase not + negated, not a stance ('MAINTAINS NOT GUILTY'), in a segment about no other + person or matter ('CO-DEFENDANT CONVICTED', 'FINED $100 FOR CONTEMPT'). Round 3: + never an offer, a bond or bail, pretrial custody, a rejected plea deal, a request or + a plea not accepted, unless the segment also reports a plea, verdict or sentence; + unknown: an outcome stated without saying which ('DISPOSED', 'JUDGMENT ENTERED') + counts too (a trial set or a new trial then ended).""" + t = _NEGATED.sub(_blank, rest) + prev_other = False + for a, b in _segments(t): + s = t[a:b] + if _CUE["outcome-other"].search(s): + prev_other = True + continue + if prev_other and _CUE["term-only-segment"].match(s): + # round 4: 'HELD IN CONTEMPT; FINED $500' -- the fine in its own segment is the + # contempt's, not the trial's outcome + continue + prev_other = False + if _CUE["later-outcome-not"].search(s) and not any( + not _negated_at(s, x.start()) and not re.search(r"\bnot\s+accepted\b", s[x.start():]) + for x in _CUE["later-outcome-strong"].finditer(s)): + continue + cues = [_CUE["later-outcome"]] + ([_CUE["later-outcome-unknown"]] if unknown else []) + for cue in cues: + for m in cue.finditer(s): + if _negated_at(s, m.start()) or _CUE["outcome-stance"].search(s[:m.start()]): + continue + return True + return False + + +def _new_trial_skip(t, m): + """None when a new-trial phrase orders a new trial of this record's charge; + else why not (the review of batch 1, R7 and round 2): 'negated' ('NO NEW TRIAL + ORDERED'), 'refused' (refused, declined, denied or only asked for), 'person' + (another person's, or -- round 3 -- another case's), 'undone' (the order itself + reversed later, or the conviction affirmed or reinstated after it), 'count' (round 3: + scoped to a count or granted in part -- the other counts' judgment stands). A new + trial limited to punishment is removed before (punishment-only).""" + if _negated_at(t, m.start()): + return "negated" + segs = _segments(t) + a, b = _seg_of(segs, m.start()) + seg = t[a:b] + if _CUE["other-person"].search(seg) or _CUE["new-trial-other-matter"].search(seg): + return "person" + asked = _CUE["new-trial-asked"].search(t[a:m.start()]) + if _CUE["new-trial-refused"].search(seg) or ( + asked and not _CUE["new-trial-granted"].search(t[m.start():b])): + return "refused" + undone = _CUE["new-trial-undone"].search(t, m.end()) + scoped = _CUE["new-trial-count"].search(seg) or _CUE["count-scope"].search(seg) + if undone and not (scoped and re.search(r"\b(?:affirmed|upheld)\b", undone.group(0))): + return "undone" + if scoped: + # (round 4: a new trial on one charge beside another charge's conviction affirmed + # is the split, not an order undone) + # round 5: a new trial on the one count the record names ('CONVICTED ON COUNT 1; NEW + # TRIAL GRANTED ON COUNT 1; ACQUITTED ON RETRIAL') splits nothing -- the retrial's + # outcome is the matter's + def norm(x): + x = re.sub(r"[^a-z0-9]+", " ", x).strip() + return re.sub(r"^(?:cts?|counts?)\b", "count", re.sub(r"^charges?\b", "charge", x)) + here = {norm(x) for x in _CUE["count-scope"].findall(seg)} + elsewhere = {norm(x) for x in _CUE["count-scope"].findall(t[:a] + " " + t[b:])} + if (len(here) == 1 and elsewhere == here and re.search(r"\d|\b(?:one|two|three|four|five|i|ii|iii)\b", + next(iter(here))) + and not re.search(r"\b(?:in\s+part|partial\w*)\b", seg)): + return None + return "count" + return None + + +def _cut_finding_labels(t): + """t with a violation, enhancement or sentencing finding removed: a not-guilty / + not-true segment right after the segment naming it, and an acquittal of a + specification or an enhancement (the review of batch 1, round 3). Never on a military + record, where a specification is the charge itself.""" + if _MILITARY_CUE.search(t): + return t + cut = _CUE["acquitted-finding"].sub(" ; ", t) + # (round 5: never the record's only verdict -- 'ACQUITTED OF BOTH SPECIFICATIONS' stays) + t = cut if re.search(r"[a-z]", cut) else t + segs = _segments(t) + for k in range(len(segs) - 1, 0, -1): + a, b = segs[k] + pa, pb = segs[k - 1] + if (_CUE["finding-bare"].match(t[a:b]) and _CUE["finding-label"].search(t[pa:pb]) + and not _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", t[pa:pb])) + # round 4: never the record's only verdict ('SPECIFICATION: NOT GUILTY') + and re.search(r"[a-z]", t[:pa] + t[b:])): + t = t[:pa] + " ; " + t[b:] + return t + + +def _other_case_conviction(t): + """t with a conviction segment that names another charge, count or case removed + when the record reports another outcome for this one ('DISMISSED - DEFENDANT + CONVICTED ON OTHER CHARGE', 'NOLLE PROS - DEF PLED GUILTY TO COUNT 1 IN RELATED + CASE'): the reason for the dismissal, not this charge's judgment (the review of batch + 1, round 3). 'PLED GUILTY; OTHER CHARGES DISMISSED' keeps its split (MA-11).""" + segs = _segments(t) + if not _CUE["outcome-word"].search(t): + return t + # round 4: 'PLED GUILTY TO A SEPARATE COUNT; CT 1 DISMISSED' names counts of this + # case -- a 'separate count' is the split-counts question, not another case + counts_named = bool(_CUE["count-scope"].search(t)) + for a, b in reversed(segs): + s = t[a:b] + m = _CUE["conviction-other-case"].search(s) + if m and re.search(r"\b(?:same|this)\s+(?:incident|conduct|facts|transaction|arrest|occurrence|episode)\b|" + r"\barising\s+(?:out\s+of|from)\b", s): + continue # round 5: a plea-down from the same incident is this matter's + if (m and not (counts_named and re.search(r"\bcounts?\b\s*$", m.group(0))) + and _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", s)) + and not _CUE["outcome-word"].search(s) + and _CUE["outcome-word"].search(t[:a] + " " + t[b:])): + t = t[:a] + " ; " + t[b:] + return t + + +def _cut_violation_warrant(t): + """t with a warrant or capias segment removed when conviction words precede it and + it -- or a segment between the conviction words and it -- names a supervision + violation, an absconding or a motion to revoke (round 4: 'CONVICTED; VOP - BENCH + WARRANT OUTSTANDING', 'CONVICTED; PROBATION; ABSCONDED; BENCH WARRANT ISSUED'): the + warrant enforces the sentence; the pending-charge names never read it. A warrant + with no conviction words before it, or with no violation named, stays.""" + segs = _segments(t) + for k in range(len(segs) - 1, 0, -1): + a, b = segs[k] + if not _CUE["warrant-word"].search(t[a:b]): + continue + before = _NEGATED_CONVICTION.sub(" ", _NEGATED.sub(" ", _fold(t[:a]))) + conv = _CONVICTION_CUES.search(before) + if not conv: + continue + if _CUE["violation-or-abscond"].search(t[a:b]) or _CUE["violation-or-abscond"].search( + t[conv.end():a]): + t = t[:a] + " ; " + t[b:] + return t + + +def _cut_other_dismissals(t): + """t with a dismissal that is not this charge's removed when conviction words stand + elsewhere (round 4): a bare dismissal word labelled by a dash or colon with costs, + fees, a warrant, a capias, a forfeiture, a bond or an order ('GUILTY - FINE $200; + DISMISSED - COSTS', 'DISMISSED - WARRANT'), and a dismissal whose object is another + case ('SEPARATE CASE DISMISSED', 'SEPARATE CASE (THEFT) DISMISSED') -- never other + charges or counts, which stay the split-counts question (MA-11).""" + segs = _segments(t) + if not _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", _NEGATED.sub(" ", _fold(t)))): + return t + cuts = [] + joined = (lambda p, q: re.search(r"[:\-–—]", t[p:q]) is not None) + for i, (a, b) in enumerate(segs): + seg = t[a:b] + m = _CUE["dismissal-word"].search(seg) + if not m: + continue + obj = " ".join(re.findall(r"[a-z0-9]+", seg[:m.start()])) + tail = " ".join(re.findall(r"[a-z0-9]+", seg[m.end():])) + if not obj and not tail: + for j in (i - 1, i + 1): + if 0 <= j < len(segs) and joined(*(segs[j][1], a) if j < i else (b, segs[j][0])): + label = " ".join(re.findall(r"[a-z0-9/]+", t[segs[j][0]:segs[j][1]])) + if _CUE["dismissal-object-non-charge"].match(label) and not _CONVICTION_CUES.search( + _NEGATED_CONVICTION.sub(" ", label)): + cuts += [(a, b), segs[j]] + elif (_CUE["dismissal-object-other-case"].search(obj + " " + tail) + and not re.search(r"\b(?:charges?|counts?)\b", obj + " " + tail) + and not _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", seg))): + cuts.append((a, b)) + for a, b in sorted(set(cuts), reverse=True): + t = t[:a] + " ; " + t[b:] + return t + + +def _relief_dismissal_joined(t): + """True when the record joins a sealing or expungement to the dismissal that came + first ('EXPUNGED AFTER DISMISSAL', 'EXPUNGEMENT GRANTED (CASE DISMISSED 2015)', + 'SEALED - NOLLE PROSEQUI') or says there was no conviction (round 4): the dismissal + is the charge's own outcome and the relief followed it -- no question, and the + dismissal entry ranks first (server._relief_yields).""" + t_rel = _NEGATED.sub(_blank, t) + m = _CUE["relief-dismissal-joined"].search(t_rel) + if m and _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", _fold( + t_rel[slice(*_seg_of(_segments(t_rel, _MAJOR_BREAK), m.start()))]))): + return False # round 5: 'SEALED - DISMISSED AFTER CONVICTION' is the relief's effect + return bool(m or (_CUE["relief-sealing"].search(t_rel) and _NEGATED_CONVICTION_WORD.search(t_rel))) + + +def relief_dismissal_joined(disposition): + """_relief_dismissal_joined on the record's words (for the server's relief ordering).""" + return _relief_dismissal_joined(record_order(disposition)[0]) + + +def set_aside_appellate(disposition): + """True when the record's words put a set-aside in one segment with an appellate word + ('CONVICTION SET ASIDE ON APPEAL; ACQUITTAL ENTERED'): the rehabilitative set-aside entry + never answers it (round 5; server._match_dispositions).""" + t = record_order(disposition)[0] + return any(_CUE["set-aside"].search(t[a:b]) and _CUE["set-aside-appellate"].search(t[a:b]) + for a, b in _segments(t)) + + +def _cut_prior_segments(t): + """t with the segments that report an earlier matter removed (round 5): one that starts + 'PRIOR', 'PREVIOUS(LY)' or 'EARLIER' ('PRIOR: CONVICTED 2010', 'PRIOR DISMISSAL 2010 NOTED', + 'PREVIOUSLY CONVICTED OF THEFT') and the segment after a bare 'PRIOR' label ('(PRIOR: + CONVICTED 2010)') -- never 'prior to', and never the record's only words. The rule + _no_charge_yields applied to the no-charge names (round 4) now applies to every record.""" + segs = _segments(t) + cuts = set() + for k, (a, b) in enumerate(segs): + s = t[a:b] + if _CUE["prior-segment"].match(s) or _PRIOR_LABEL.match(s): + cuts.add(k) + if _PRIOR_LABEL.match(s) and k + 1 < len(segs): + cuts.add(k + 1) + elif _CUE["prior-label-tail"].search(s) and k + 1 < len(segs): + cuts.add(k + 1) + cuts.add((k, "tail")) + if not cuts: + return t + kept = [t[a:b] for k, (a, b) in enumerate(segs) if k not in cuts] + if not any(re.search(r"[a-z]", x) for x in kept): + return t + for k in sorted((k for k in cuts if isinstance(k, int)), reverse=True): + a, b = segs[k] + t = t[:a] + " ; " + t[b:] + for k, _ in (c for c in cuts if isinstance(c, tuple)): + a, b = segs[k] + t = t[:a] + _CUE["prior-label-tail"].sub(" ", t[a:b]) + t[b:] + return t + + +def _set_aside_legal_error(t): + """t with a set-aside for a stated legal-error ground (habeas, a writ, s. 2255, art. + 11.07, ineffective assistance, voidness) and no appellate or rehabilitative word read as + the reversal entry's words (round 5: 'CONVICTION SET ASIDE ON HABEAS CORPUS' is a vacated + conviction, never the rehabilitative row and never the set-aside question the record + answers): the set-aside phrase reads 'conviction quashed' when no outcome follows it, and + goes when the record states what followed ('...; CHARGES DISMISSED', '...; NOT GUILTY').""" + t_sa = _NEGATED.sub(" ", t) + if not (_CUE["set-aside"].search(t_sa) and _CUE["set-aside-legal-error"].search(t_sa) + and not _CUE["set-aside-appellate"].search(t_sa) + and not _CUE["set-aside-rehabilitative"].search(t_sa)): + return t + m = _CUE["set-aside"].search(t) + a, b = _seg_of(_segments(t), m.start()) + later = _later_outcome(t[b:]) or _CUE["later-acquittal"].search(t, b) or _CUE["dismissal-word"].search(t, b) + return t[:a] + (" ; " if later else " ; conviction quashed ; ") + t[b:] + + +def _cut_pending_other(t): + """t with a new, separate or unrelated charge or case pending removed after conviction + words (round 5: 'CONVICTED; 2 YRS PROBATION; NEW CHARGE PENDING IN ANOTHER COUNTY' -- the + pending matter is another case's, the conviction decides this record), and a bare 'CASE + PENDING' after conviction words and a stayed sentence.""" + segs = _segments(t) + for k in range(len(segs) - 1, 0, -1): + a, b = segs[k] + s = t[a:b] + if not (_CUE["pending-other-matter"].search(s) or _CUE["case-pending-bare"].match(s)): + continue + before = _NEGATED_CONVICTION.sub(" ", _NEGATED.sub(" ", _fold(t[:a]))) + if _CONVICTION_CUES.search(before): + t = t[:a] + " ; " + t[b:] + return t + + +def record_order(disposition): + """(words, flags): the record's words after the routing rules of the + record-order cue table (lower-cased, shorthand expanded, accent-folded, + punctuation kept), and the ids of every rule the wording raised + (DISPOSITION_FLAGS order). Pure; memoised (one lookup, so a concurrent + clear cannot fall between a membership test and the read).""" + key = disposition or "" + hit = _RECORD_ORDER_CACHE.get(key) + if hit is not None: + return hit + t = _accent_fold(expand_disposition(key)) + flags = set() + + def cut(pattern, flag, repl=" ; "): + nonlocal t + new = pattern.sub(repl, t) + if new != t: + if flag: + flags.add(flag) + t = new + cut(_CUE["plea-not-guilty"], "plea-not-guilty") + cut(_CUE["no-disposition"], "no-disposition") + # another person's outcome, and a violation found not proven, are not this record's + cut(_CUE["other-person-subject"], None) + for m in reversed(list(_CUE["other-person-object"].finditer(t))): + a, b = _seg_of(_segments(t), m.start()) + t = t[:a] + " ; " + t[b:] + cut(_CUE["violation-not-found"], None) + # a violation, enhancement or sentencing finding in its own segment (round 3) + t = _cut_finding_labels(t) + # an insanity plea or defence rejected or withdrawn is no verdict (round 5), and a segment + # reporting an earlier matter ('PRIOR: CONVICTED 2010') is not this record's + cut(_CUE["insanity-rejected"], None) + t = _cut_prior_segments(t) + nodisp = "no-disposition" in flags + # a sentencing continuance ('SENTENCE DEFERRED PENDING PSI', 'SENTENCING DEFERRED TO + # 03/01', 'SENTENCE DEFERRED 60 DAYS') is procedure when the record then reports the + # sentence imposed (round 4): cut, so the conviction words decide + # (round 5: in either word order -- 'DEFERRED SENTENCE PENDING PSI' --, and after conviction + # words a bare term in a later segment ('GUILTY - SENTENCE DEFERRED PENDING PSI - 30 DAYS') + # is the sentence imposed; a sentence deferred pending appeal is the appeal pending) + conts = list(_CUE["sentencing-continuance"].finditer(t)) + list( + _CUE["sentencing-continuance-reversed"].finditer(t)) + for m in sorted(conts, key=lambda x: x.start(), reverse=True): + segs = _segments(t) + seg_end = _seg_of(segs, m.start())[1] + if re.search(r"\bpending\s+(?:the\s+)?appeal\b", m.group(1) or ""): + t = t[:m.start()] + " ; appeal pending ; " + t[m.end():] + continue + convicted_before = bool(_CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", _fold(t[:m.start()])))) + later = _CUE["sentence-later-verb"] if not m.group(1) else _CUE["sentence-later-imposed"] + term_after = convicted_before and m.group(1) and any( + a >= seg_end and _CUE["term-bare"].match(t[a:b]) for a, b in segs) + if later.search(t, seg_end) or term_after: + t = t[:m.start()] + " ; " + t[m.end():] + # a warrant on a supervision violation or an absconding after conviction words is + # the sentence's enforcement, not a pending charge (round 4: 'CONVICTED; VOP - BENCH + # WARRANT OUTSTANDING') + t = _cut_violation_warrant(t) + # another matter's charge pending, and a bare 'case pending' after a stayed sentence, after + # conviction words (round 5) + t = _cut_pending_other(t) + # a relief application pending (round 5): no relief yet -- a pardon, clemency or commutation + # pending presupposes the conviction, which stays; an expungement or sealing petition + # pending says nothing about the disposition + for m in reversed(list(_CUE["relief-pending"].finditer(t))): + t = t[:m.start()] + (" ; conviction ; " if m.group(1) else " ; ") + t[m.end():] + # a Commonwealth appeal against the conviction allowed is the conviction quashed on + # appeal (round 4; Criminal Appeal Act 1968 s. 2(2) form of words); an appeal allowed + # with no object may be against the sentence only: the reversal entry is one reading + # and the question is asked (flag appeal-allowed) + cut(_CUE["judgment-for-costs"], None) + # (round 5: leave to appeal allowed or granted is an appeal pending, refused is nothing; a + # prosecution appeal allowed against an acquittal with a conviction stated after it takes + # the acquittal and the appeal words with it, and the conviction decides) + t = _CUE["leave-to-appeal"].sub( + lambda m: " ; appeal pending ; " if m.group(1) in ("allowed", "granted") else " ; ", t) + for m in reversed(list(_CUE["prosecution-appeal"].finditer(t))): + after = _NEGATED_CONVICTION.sub(" ", _NEGATED.sub(" ", _fold(t[m.end():]))) + if _CONVICTION_CUES.search(after) and re.search(r"\b(?:acquit\w*|not guilty)\b", t[:m.start()]): + t = " ; " + t[m.end():] + cut(_CUE["appeal-against-conviction-allowed"], None, " ; conviction quashed on appeal ; ") + for m in reversed([m for m in _CUE["appeal-allowed"].finditer(t) if not _negated_at(t, m.start())]): + varied = _CUE["sentence-varied"].match(t[m.end():]) + if varied: + # round 5: 'APPEAL ALLOWED - SENTENCE VARIED': the conviction stands + t = t[:m.start()] + " ; conviction ; " + t[m.end() + varied.end():] + continue + if _CUE["appeal-outcome-after"].search(t, m.end()): + continue + flags.add("appeal-allowed") + t = t[:m.start()] + " ; conviction quashed on appeal ; " + t[m.end():] + # a refusal bound to relief, a programme or an offer: those words grant nothing -- + # round 4: what a refused motion attacked stays ('conviction' for a refused motion to + # set aside the conviction; the relief word for a refused motion against the relief) + for start, end, keep in sorted(_refused_spans(t), reverse=True): + flags.add("refused-relief") + t = t[:start] + (f" ; {keep} ; " if keep else " ; ") + t[end:] + # a dismissal labelled costs, a warrant or a bond, or of another case, beside + # conviction words (round 4) -- before the non-charge cut, so 'DISMISSED - CAPIAS + # WITHDRAWN' goes as one labelled non-charge dismissal (round 5) + t = _cut_other_dismissals(t) + # a dismissal of a violation, a motion, costs or counsel is not the charge's + cut(_CUE["non-charge-dismissal"], None) + # a civil order or a peace bond dismissed beside another disposition (round 3) + for m in reversed(list(_CUE["non-charge-dismissal-order"].finditer(t))): + rest = _fold(t[:m.start()] + " " + t[m.end():]) + if _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", rest)) or _NON_CONVICTION_CUES.search(rest): + t = t[:m.start()] + " ; " + t[m.end():] + # a conviction on another charge or case given as a dismissal's reason (round 3) + t = _other_case_conviction(t) + # a new trial or reversal limited to punishment leaves the conviction standing + cut(_CUE["punishment-only"], None) + # a new-trial motion denied on one count and granted on another (round 5): the denied + # count's conviction stands, the granted count is the split + t = _CUE["new-trial-split-motion"].sub( + lambda m: f"conviction on {m.group(1)} stands ; new trial granted on {m.group(2)}", t) + # a new trial ordered: the record reads from it -- pending, unless the record + # says how the new trial ended (the review of batch 1, R7); scoped to a count or + # granted in part, the other counts' judgment stands (round 3) + for m in reversed(list(_CUE["new-trial"].finditer(t))): + why = _new_trial_skip(t, m) + if why in ("refused", "person"): + a, b = _seg_of(_segments(t), m.start()) + t = t[:a] + " ; " + t[b:] + elif why == "undone": + segs = _segments(t) + a = _seg_of(segs, m.start())[0] + undone = _CUE["new-trial-undone"].search(t, m.end()) + # round 4: an order undone by reinstating or affirming the conviction keeps one + # conviction word ('MOTION FOR NEW TRIAL GRANTED; STATE APPEALED; REVERSED; + # CONVICTION REINSTATED' had lost its only judgment) + keep = (" ; convicted ; " if re.search(r"\b(?:reinstated|affirmed|upheld)\b", undone.group(0)) + else " ; ") + t = t[:a] + keep + t[_seg_of(segs, undone.start())[1]:] + elif why == "count" and _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", _fold(t))): + flags.add("split-counts") + nt = next((m for m in _CUE["new-trial"].finditer(t) if _new_trial_skip(t, m) is None), None) + if nt: + flags.add("new-trial") + rest = t[nt.end():] + rd = _CUE["retry-declined"].search(rest) + # K1 (2026-09-23 -- J1-L8): the declination collapses the WHOLE record to 'not + # prosecuted', so nothing else in the remainder may be another count's own outcome. + # Before this the count scope was looked for only inside the declination's own + # segment, and 'convicted on both counts; new trial ordered; Crown declined to + # retry; 2 years on count 2' was replaced by ' ; not prosecuted ; ' -- the 2-year + # term on the count the new trial never touched was erased and the entry answered + # NO CONVICTION. The whole remainder is searched now, and a count scope raises + # split-counts instead. + other_matter = bool(_CUE["count-scope"].search(rest)) or any( + _CUE["term-bare"].match(rest[a:b]) for a, b in _segments(rest)) + if rd and not _later_outcome(rest[rd.end():]) and not other_matter: + t = " ; not prosecuted ; " # the prosecution declined to try it again (round 3) + else: + if rd: + # round 4: a declination on one count beside a conviction at retrial on + # another is split-counts; a later conviction or plea is read + if _CUE["count-scope"].search(rest): + flags.add("split-counts") + ended = True + else: + ended = _later_outcome(rest, unknown=True) or _CUE["retrial-term"].search(rest) + if ended and not _later_outcome(rest) and not _CUE["retrial-term"].search(rest): + # round 4: an outcome stated without saying which ('JUDGMENT ENTERED') + # ends the new trial with the outcome unknown -- never the conviction + # entry's name + rest = _CUE["later-outcome-unknown"].sub(" ; ", rest) + t = (" ; " if ended else " pending trial ; ") + rest + # a count-scoped appeal (round 5): one count's conviction quashed or reversed beside another + # count affirmed, or a conviction 'AFFIRMED IN PART, REVERSED IN PART' -- the counts split + if _CUE["affirmed-in-part"].search(t) or ( + _CUE["count-affirmed"].search(t) and any( + _CUE["count-undone"].search(t[a:b]) and _CUE["count-scope"].search(t[a:b]) + for a, b in _segments(t))): + flags.add("split-counts") + # a hung jury or mistrial, then the prosecution declining to retry (round 5): not prosecuted + if not nt: + rd = _CUE["retry-declined"].search(t) + if rd and _CUE["mistrial"].search(t, 0, rd.start()) and not _later_outcome(t[rd.end():]): + t = " ; not prosecuted ; " + # a trial set: pending, unless a later outcome is reported (R1) -- round 3: or a + # judgment came before it, it names a count beside a conviction (split-counts), or it + # is a violation or revocation proceeding's trial + ts = _CUE["trial-set"].search(t) + if ts: + segs = _segments(t) + a, b = _seg_of(segs, ts.start()) + seg = t[a:b] + judged_before = _CUE["judgment-before"].search(_NEGATED_CONVICTION.sub(" ", _NEGATED.sub(" ", t[:a]))) + if _CUE["count-scope"].search(seg) and _CONVICTION_CUES.search( + _NEGATED_CONVICTION.sub(" ", _fold(t[:a] + " " + t[b:]))): + flags.add("split-counts") + t = _CUE["trial-set"].sub(" ; ", t) + elif (_CUE["violation-proceeding"].search(seg) or judged_before + or _later_outcome(t[ts.end():], unknown=True)): + if not (_CUE["violation-proceeding"].search(seg) or judged_before + or _later_outcome(t[ts.end():])): + # round 4: 'TRIAL SET; JUDGMENT ENTERED' -- the setting ended, the outcome + # unknown: the unknown-outcome words go too, never the conviction entry's + # name 'judgment entered' + t = t[:ts.end()] + _CUE["later-outcome-unknown"].sub(" ; ", t[ts.end():]) + t = _CUE["trial-set"].sub(" ; ", t) + else: + cut(_CUE["trial-set"], "trial-set", " pending trial ") + # tried as an adult -- never a transfer that did not take effect, and not when + # the case went back to juvenile court (R3, round 2) + t = _adult_court(t, flags) + # (round 4: the term or fine stayed keeps its words; a stay of execution of the + # sentence -- Minnesota -- keeps nothing, the conviction words decide) + cut(_CUE["stayed-term"], "stayed-term", lambda m: m.group(1) or " ") + # a penal order objected to and then acquitted: the outcome is read + po = _penal_order_state(t) + if isinstance(po, tuple): + t = t[po[1]:] + # a judgment the record says followed the revocation of the programme, in the + # judgment's own segment (round 4): the judgment is read, the programme words go + jr = _CUE["judgment-upon-revocation"].search(t) + if jr and not _negated_at(t, jr.start()): + flags.add("revoked-adjudicated") + t = t[:jr.start()] + " ; " + jr.group(1) + " ; " + t[jr.end():] + hit = _revoked_then_adjudicated(t) + if hit is not None: + flags.add("revoked-adjudicated" if hit[1] in ("guilt", "mta") else "later-judgment") + t = t[hit[0]:] + segs = _segments(t) + if segs: + a, b = segs[0] + if re.sub(r"[^a-z]+", "", _CUE["revocation-filler"].sub(" ", t[a:b])) == "adjudicated": + # a bare 'ADJUDICATED' after the revocation is the adjudication of guilt + # (round 3: never the Florida juvenile name 'adjudicated') + t = t[:a] + re.sub(r"\badjudicated\b", "adjudicated guilty", t[a:b], count=1) + t[b:] + elif hit[1] == "mta": + # round 5: a granted motion to adjudicate guilt is the adjudication (Tex. Code + # Crim. Proc. art. 42A.108) -- the motion's words read 'adjudicated guilty' + m = _CUE["programme-motion-granted"].search(t, a, b) + t = t[:m.start()] + "adjudicated guilty" + t[m.end():] + elif _CUE["term-bare"].match(t[a:b]): + # round 5: a bare term after the programme itself was revoked is the sentence + # imposed ('PBJ REVOKED; 30 DAYS' reads 'sentenced 30 days') + t = t[:a] + "sentenced " + t[a:].lstrip() + else: + split_deferral = _deferral_count_split(t) + if split_deferral: + flags.add("split-counts") + hit = None if split_deferral else _later_judgment(t) + if hit is not None and hit[1] == "read": + flags.add("later-judgment") + t = t[hit[0]:] + elif hit is not None: + for start, end in sorted(hit[2], reverse=True): + t = t[:start] + " ; " + t[end:] + if hit[1] == "question": + flags.add("later-conviction") + # a set-aside for a stated legal-error ground reads as a vacated conviction (round 5) + t = _set_aside_legal_error(t) + if _CUE["sentencing-pending"].search(t) and not _CUE["sentence-imposed"].search( + _CUE["sentencing-pending"].sub(" ", t)): + cut(_CUE["sentencing-pending"], "sentencing-pending") + # the question rules read the words as the cues do (negated words dropped) + folded = _NEGATED.sub(" ", _fold(t)) + live = _NEGATED_CONVICTION.sub(_blank, folded) + conv = _CONVICTION_CUES.search(live) + if _split_counts(t, folded, live, conv, nodisp): + flags.add("split-counts") + # conviction words, then a dismissal: which (a deferral's own plea words -- + # 'DEFERRED ACCEPTANCE OF GUILTY PLEA' -- are not conviction words here) + # (round 5: a withdrawn plea before a dismissal keeps the question -- the critic's critical + # row 'GUILTY PLEA WITHDRAWN, CHARGE DISMISSED' asks whether judgment was entered before + # the dismissal; the adversarial lens read it as a plain dismissal: unsettled, MA-12) + t_live = _CUE["deferral-plea-phrase"].sub(_blank, _NEGATED_CONVICTION.sub(_blank, _NEGATED.sub(_blank, t))) + conv_t = _CONVICTION_CUES.search(t_live) + if (conv_t and "split-counts" not in flags + and not _CUE["appeal-word"].search(t_live, conv_t.start())): + after = _dismissal_after(t, t_live, conv_t) + if after == "split": + flags.add("split-counts") + elif after == "question": + flags.add("post-conviction-dismissal") + # a set-aside with no rehabilitative ground or statute named, and no appellate word + # (the reversal entry reads those): the question -- a legal-error ground (habeas, a + # writ, ineffective assistance, voidness) keeps it, so no rehabilitative row answers + # a vacatur (round 4) + t_sa = _NEGATED.sub(" ", t) + if (_CUE["set-aside"].search(folded) and not _CUE["set-aside-appellate"].search(t_sa) + and (_CUE["set-aside-legal-error"].search(t_sa) + or not _CUE["set-aside-rehabilitative"].search(t_sa))): + flags.add("set-aside-ambiguous") + if _penal_order_state(t) == "open": + flags.add("penal-order-not-final") + # a sealing or expungement, then a dismissal, and no conviction words: was there a + # conviction before the relief? (the review of batch 1, round 3) -- not when the + # words join the relief to the dismissal that came first, or say 'no conviction' + # (round 4: 'EXPUNGED AFTER DISMISSAL', 'EXPUNGEMENT GRANTED (CASE DISMISSED 2015)') + t_rel = _NEGATED.sub(_blank, t) + rel = _CUE["relief-sealing"].search(t_rel) + if (rel and not conv_t and _CUE["dismissal-word"].search(t_rel, rel.end()) + and not _CUE["dismissal-word"].search(t_rel, 0, rel.start()) + and not _relief_dismissal_joined(t)): + flags.add("relief-then-dismissal") + # the matter brought back after it was dropped, and no outcome after that + segs = _segments(t) + cands = list(_CUE["reinstituted"].finditer(t)) + list(_CUE["reinstituted-extra"].finditer(t)) + for k in range(1, len(segs)): + a, b = segs[k] + if (_CUE["reinstated-bare"].match(t[a:b]) + and _CUE["dismissal-word"].search(t[segs[k - 1][0]:segs[k - 1][1]])): + cands.append(re.compile(r"re-?instated").search(t, a, b)) + elif _CUE["indicted-bare"].match(t[a:b]) and _CUE["indictment-withheld"].search(t[:a]): + # round 4: a bare 'INDICTED' after a no-bill or a withheld indictment brings + # the matter back ('NO BILL; INDICTED' asks how it ended) + cands.append(re.compile(r"\bindicted\b").search(t, a, b)) + for m in sorted(cands, key=lambda x: x.start()): + if not re.sub(r"[^a-z0-9]+", "", t[:m.start()]): + continue + k = next((i for i, (a, b) in enumerate(segs) if a <= m.start() < b), None) + if k is None: + continue + a, b = segs[k] + seg, nxt = t[a:b], (t[segs[k + 1][0]:segs[k + 1][1]] if k + 1 < len(segs) else "") + if (_negated_at(t, m.start()) or _CUE["reinstituted-negated"].search(t[a:m.start()]) + or _CUE["reinstituted-modal"].search(t[max(0, m.start() - 40):m.start()]) + or _CUE["reinstituted-history"].match(t[m.end():]) + or _CUE["other-person"].search(seg) or _CUE["other-person-family"].search(seg) + or _CUE["reinstituted-not"].search(seg) + or (_CUE["reinstituted-not"].search(nxt) and re.search( + r"\b(?:re-?fil\w*|re-?instat\w*|re-?indict\w*|re-?charg\w*)\b", nxt)) + or _CUE["other-charge"].search(t[m.end():]) + or (re.search(r"(?:indicted|charged|prosecuted)$", m.group(0)) + and _CUE["later-charged-other"].search(t[m.end():b]))): + continue + if re.search(r"\b(?:pending|awaiting\s+trial|open\s+case|case\s+open|set\s+for\s+trial|trial\s+set)\b", + t[m.end():]): + # round 5: the matter brought back and said to be pending ('DISMISSED; REFILED AS + # CASE NO. 2020-CR-100; PENDING') reads from the back word -- a pending charge + t = t[m.start():] + break + if not _later_outcome(t[m.end():]): + flags.add("reinstituted") + break + out = (t, tuple(f for f in DISPOSITION_FLAGS if f in flags)) + if len(_RECORD_ORDER_CACHE) > 4096: + _RECORD_ORDER_CACHE.clear() + _RECORD_ORDER_CACHE[key] = out + return out + + +def disposition_flags(disposition): + """The record-order flags the disposition's words raise (record_order).""" + return list(record_order(disposition)[1]) + + +def _cue_text(disposition): + return strip_appeal_upheld(_NEGATED.sub(" ", _fold(record_order(disposition)[0]))) + + +def disposition_cues(disposition, without=()): + """'conviction' when the record's own words say it was convicted and + nothing in them names a non-conviction disposition; 'non-conviction' + when a non-conviction phrase appears (negated ones -- 'conviction not + set aside' -- do not count); None when neither. A negated conviction + ('NOT CONVICTED', 'NO CONVICTION') is no conviction cue (round 4). + without: phrases (folded) the server has set aside -- a no-charge name that + yielded to conviction words in another segment (server._no_charge_yields) -- + blanked before the cues read the words, so 'DA DECLINED TO FILE FELONY; + MISDEMEANOR FILED; CONVICTED' reads its conviction (round 4).""" + t = _cue_text(disposition) + for w in without: + words = _fold(w).split() + if words: + t = re.sub(r"\b" + r"[^a-z0-9]+".join(re.escape(x) for x in words) + r"\b", " ", t) + if _NON_CONVICTION_CUES.search(t): + return "non-conviction" + if _CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", t)): + return "conviction" + return None + + +def programme_running(disposition): + """True when the record's words say a programme (probation, a deferral, a + diversion, an agreement) is still running and nothing in them says it + ended (disposition roadmap T1.10).""" + t = _accent_fold(disposition or "") + m = _CUE["programme-running"].search(t) + if not m: + return False + rest = t[:m.start()] + " " + t[m.end():] + return not _CUE["programme-ended"].search(rest) + + +def duress_words(disposition): + """True when the record's words say the plea or the act was coerced (the + critic's Rodriguez Anzola item).""" + return bool(_CUE["duress-words"].search(_accent_fold(disposition or ""))) + + +def has_conviction_words(disposition): + """True when the record's words include a conviction cue ('guilty', + 'convicted', 'sentenced') that nothing negates -- whatever else they + say. A bare 'pending' yields to it ('found guilty, sentence stayed + pending appeal' is not a pending charge).""" + return bool(_CONVICTION_CUES.search(_NEGATED_CONVICTION.sub(" ", _cue_text(disposition)))) + + +def disposition_match_text(disposition): + """The words the curated entries are matched against: shorthand + expanded, negated non-conviction words dropped, and a phrase that is + answered by a note instead of an entry ('record suspension') removed.""" + t = _cue_text(disposition) + t = re.sub(r"\brecord suspension\w*\b", " ", t) + # a cancelled or negated conviction is answered by the cue, not by an + # entry its leftover words happen to share ('no ... recorded'; round 4: 'NOT + # CONVICTED', 'NO CONVICTION' never reach the conviction entry's names) + t = _CANCELLED_CONVICTION.sub(" ", t) + t = _NEGATED_CONVICTION_WORD.sub(" ", t) + return re.sub(r"\s+", " ", t).strip() + + +def disposition_notes(disposition): + """Step-1 notes the record's words call for, whatever entry matches.""" + notes = [] + if re.search(r"\brecord suspension", _fold(expand_disposition(disposition))): + notes.append(RECORD_SUSPENSION_NOTE) + return notes + + +# ---- the Saini checklist (roadmap T2.8; track A step A9) -------------------- +# Recognising a foreign discharge or pardon: Canada (MCI) v Saini, 2001 FCA 311, +# para 24, with the motivating-considerations and seriousness points the FCA +# and ENF 3 s. 10.9 add. Every quoted phrase is pinned to its corpus chunk by +# tests/test_pardons_resolvers.py. WHICH disposition entries print the +# checklist, and the entry's note under each item, are curated +# (us_dispositions.json 'pardon_test', optional; held for review) -- the +# approved baseline has none, so a composed release prints no checklist. +SAINI_QUOTES = { + "para24": ("fca-31531-14", "three elements must be established before a foreign " + "discharge or pardon may be recognized"), + "system": ("fca-31531-14", "the foreign legal system as a whole must be similar to " + "that of Canada"), + "law": ("fca-31531-14", "the aim, content and effect of the specific foreign law " + "must be similar to Canadian law"), + "reason": ("fca-31531-14", "there must be no valid reason not to recognize the effect " + "of the foreign law"), + "similar": ("fca-31531-16", 'The systems must be "similar" not just "somewhat similar"'), + "evidence": ("fca-31531-16", "the similarity of the systems must normally be proved " + "by evidence"), + "erase": ("enf-3-10-9", "whether the effect of the pardon is to erase a conviction or " + "merely recognize that rehabilitation has taken place"), + "latter": ("enf-3-10-9", "In the latter case, the applicant is inadmissible"), + "motivation": ("enf-3-10-9", "Canadian courts are not bound by a foreign pardon in which " + "there is an absence of evidence as to the motivating " + "considerations"), + "certified": ("fca-31531-32", "This Court answers no to the first certified question"), + "seriousness": ("fca-31531-27", "the seriousness of the offence can be considered under " + "this third requirement"), + "onus": ("fca-31531-29", "the onus is on that person to do so"), + "rare": ("fca-31531-30", "Foreign pardons should only be recognized in rare situations"), +} +SAINI_SOURCES = ( + "Sources: the Saini FCA reasons in CanLex (canlex_search_legislation, doc_type " + "'caselaw', query 'Saini 2001 FCA 311'): chunks fca-31531-14 (para 24), " + "fca-31531-16 (paras 29-30), fca-31531-27 (para 43), fca-31531-29 (para 45), " + "fca-31531-30 (para 47), fca-31531-31 and fca-31531-32 (the certified questions); " + "ENF 3 s. 10.9 'Foreign pardons' (enf-3-10-9; official, 2022-03-03); IRB Legal " + "Services, Removal Order Appeals ch. 8 ss. 16-18 'Discharges and Pardons', " + "'Effect of a Discharge', 'Effect of a Pardon' (irb-ls-roa8-16, irb-ls-roa8-17, " + "irb-ls-roa8-18-p1 to irb-ls-roa8-18-p8; an archived 2009 paper); ENF 14 s. 27 " + "'Pardons outside Canada' (enf-14-27; a withdrawn 2008 ATIP copy).") + + +def saini_checklist(entry): + """The Saini checklist lines for a disposition entry that carries the + curated 'pardon_test' (an object whose optional keys -- system, law, + reason, motivation, seriousness -- hold the entry's note for that item); + [] for any other entry. The items are the officer's to answer: CanLex + records no answer.""" + test = (entry or {}).get("pardon_test") + if not isinstance(test, dict): + return [] + q = {k: v[1] for k, v in SAINI_QUOTES.items()} + + def item(key, text): + note = test.get(key) + return f"[ ] {text}" + (f" For this entry: {note}" if note else "") + return [ + ("SAINI CHECKLIST -- a foreign discharge or pardon is recognised only when " + f"all three elements are met: '{q['para24']}' (Canada (MCI) v Saini, 2001 " + f"FCA 311, para 24). The person relying on it bears the onus ('{q['onus']}', " + f"para 45), and '{q['rare']}' (para 47). Record the answer to each item; " + "CanLex does not answer them."), + item("system", f"1. The legal system -- '{q['system']}': '{q['similar']}', and " + f"'{q['evidence']}' (paras 29-30)."), + item("law", f"2. The specific foreign law -- '{q['law']}'. ENF 3 s. 10.9: examine " + f"'{q['erase']}'. '{q['latter']}' and the route is rehabilitation " + "(IRPR s. 17)."), + item("reason", f"3. No valid reason to refuse -- '{q['reason']}'."), + item("motivation", "Motivating considerations -- what led to the grant. ENF 3 " + f"s. 10.9: '{q['motivation']}' (Saini, certified question 1: " + f"'{q['certified']}', para 51)."), + item("seriousness", f"Seriousness of the offence -- '{q['seriousness']}' (para 43)."), + SAINI_SOURCES, + ] + + +# ---- element resolvers (roadmap T2.10; track A step A9) --------------------- +# A pairing whose branch 'depends on the elements' may carry curated +# 'resolvers' (us_equivalency.json, optional; held for review). A resolver +# names the fact that decides which Canadian provision the record is, the +# question, the Canadian element as a verbatim statute fragment with its +# corpus chunk id, and one outcome per answer -- each outcome that resolves is +# tied to one of the pairing's own verify rows (section and maximum). The code +# only reads the answer: +# 1. an explicit answer in the officer's facts, written ': ' +# (several answers separated by commas); +# 2. otherwise an outcome's cue words -- words of the Canadian element +# ('weapon', 'bodily harm', 'chok...') -- in the record's offence words or +# in the facts. A cue the words negate or doubt ('no weapon', 'did not +# choke', 'alleged strangulation') does not count. +# An outcome whose element came into force after the offence ('from') is +# applied only when the offence date is given and on or after it. Answers +# pointing to different provisions, an answer naming no provision, or no +# answer at all leave the branch 'depends', with a 'needs' entry naming the +# fact, the question and every option. A resolver never yields 'no +# equivalent': an outcome without a maximum resolves nothing. +_NOT_ASSERTED = frozenset({ + "no", "not", "without", "never", "nor", "neither", "didn't", "didnt", "wasn't", + "wasnt", "isn't", "isnt", "denied", "denies", "deny", "alleged", "allegedly", + "possibly", "maybe", "unclear", "unknown", "unconfirmed"}) +_CLAUSE_BREAK = frozenset({"but", "and", "then", "however", "although", "though", + "while", "yet"}) +# An element's words found in the OFFENCE text only (wave 2 verifier): +# printed as a suggestion, the question stays open. +OFFENCE_WORDS_NOTE = ( + "'{fact}': the record's offence words ('{cues}') suggest '{answers}' ({labels}) -- not " + "settled on the offence's name alone: whether the foreign offence requires that element " + "is the Hill comparison, and a foreign statute may be broader than the Canadian " + "element. Confirm it from the foreign statute or the facts and answer in the facts as " + "'{fact}: '") +_WORDS_AND_BREAKS = re.compile(r"[a-z0-9']+|[,;.:!?()\n]") + + +def _plain(text): + t = unicodedata.normalize("NFKD", (text or "").lower().replace("\u2019", "'")) + return "".join(ch for ch in t if not unicodedata.combining(ch)) + + +def _asserted(text, start): + """False when the words just before `start` negate or doubt what follows + ('no weapon', 'did not choke', 'alleged strangulation'): up to three words + back, stopping at punctuation or a conjunction ('no injuries but choked + her' asserts the choking).""" + seen = 0 + for tok in reversed(_WORDS_AND_BREAKS.findall(text[:start])): + if not (tok[0].isalnum() or tok[0] == "'") or tok in _CLAUSE_BREAK: + return True + if tok in _NOT_ASSERTED: + return False + seen += 1 + if seen >= 3: + return True + return True + + +def element_cues_found(cues, text): + """The cues asserted in `text` (a cue matches at the start of a word, so + 'chok' finds 'choked' and 'choking').""" + text = _plain(text) + found = [] + for cue in cues or []: + c = _plain(cue).strip() + if not c: + continue + for m in re.finditer(r"(?: ' written for this pairing's + resolvers removed, and any bare fact id -- a fact's own name + ('harm-weapon-choking') is never read as an element's words.""" + t = _plain(facts_text) + for r in pairing.get("resolvers") or []: + fact = r["fact"].lower() + t = re.sub(r"(?' ({answers})"), + "options": opts, "elements": _element_items(resolver)} + + +def resolve_elements(pairing, offense, facts_text=None, at=None, dated=False, + status="foreign-national"): + """Run the pairing's curated resolvers (see the comment above). + + at: the date the maximum is assessed at; dated: True only when `at` is the + offence date itself -- an element in force only from a later date is + applied only then. Returns {"resolved": None or {fact, answer, answers, + by ('facts-answer' | 'offence-words' | 'facts-words'), cues, label, + section, max_years, current_max, elements, note}, "needs": [...], + "notes": [...]} -- the first resolver that settles the provision wins.""" + out = {"resolved": None, "needs": [], "notes": []} + for r in pairing.get("resolvers") or []: + outcomes = r.get("outcomes") or [] + by_answer = {o["answer"]: o for o in outcomes} + written = resolver_answers(r["fact"], facts_text) + unknown = [a for a in written if a not in by_answer] + if unknown: + out["notes"].append(f"'{r['fact']}': answer(s) {', '.join(unknown)} not " + f"recognised (answers: {' | '.join(by_answer)})") + answers = [a for a in written if a in by_answer] + chosen, by, cues = [], None, [] + if answers: + chosen, by = [by_answer[a] for a in answers], "facts-answer" + else: + # the facts first: an outcome the facts' words assert is read from the facts + words = (("facts-words", _without_answers(pairing, facts_text)), + ("offence-words", analyse(offense).text)) + srcs = set() + for o in outcomes: + for src, text in words: + hit = element_cues_found(o.get("cues"), text) + if hit: + chosen.append(o) + cues += hit + srcs.add(src) + break + by = ("facts-words" if "facts-words" in srcs else "offence-words") if srcs else None + # wave 2 verifier (principle 5): the offence's NAME alone ('false + # personation') suggests an answer but never settles it -- a foreign + # statute may be broader than the Canadian element (Hill); only the + # facts, or an explicit ': ', settle. A curated + # outcome may say otherwise ('cue_settles': true, held for review). + if srcs == {"offence-words"} and not all(o.get("cue_settles") is True + for o in chosen): + out["notes"].append(OFFENCE_WORDS_NOTE.format( + fact=r["fact"], cues="', '".join(dict.fromkeys(cues)), + answers="' or '".join(o["answer"] for o in chosen), + labels="; ".join(f"{o['label']}: {o.get('section') or 'none'}" + for o in chosen))) + need = _resolver_need(pairing, r, at, status) + need["suggested"] = [o["answer"] for o in chosen] + out["needs"].append(need) + continue + applied = [] + for o in chosen: + frm = parse_date(o["from"]) if o.get("from") else None + if frm and not (dated and at is not None and at >= frm): + when = ("the offence was committed before then" if dated and at is not None + else "the offence date is not given: give offence_date") + out["notes"].append(f"{o['label']} -- {o.get('section')} counts only for an " + f"offence on or after {o['from']}; {when}" + + (f". {o['note']}" if o.get("note") else "")) + continue + applied.append(o) + provisions = {(o.get("section"), outcome_max(o, at)) for o in applied} + resolving = [o for o in applied if o.get("max_years") is not None] + if applied and len(provisions) == 1 and resolving: + o = resolving[0] + mx = outcome_max(o, at) + if at is None and o.get("history"): + first = sorted(o["history"], key=lambda h: h["until"])[0] + out["notes"].append(f"{o.get('section')}: the maximum was " + f"{first['max_years']} years until {first['until']} (Tran) " + "-- give offence_date") + out["resolved"] = { + "fact": r["fact"], "answer": o["answer"], + "answers": answers or [x["answer"] for x in applied], "by": by, + "cues": list(dict.fromkeys(cues)), "label": o["label"], + "section": o.get("section"), "max_years": mx, "current_max": o["max_years"], + "elements": _element_items(r, o["answer"]), "note": o.get("note")} + return out + if applied and not resolving: + for o in applied: + out["notes"].append(f"'{r['fact']}: {o['answer']}' -- {o['label']}: " + + (o.get("note") or "no provision this pairing names")) + continue + if len(provisions) > 1: + out["notes"].append(f"'{r['fact']}': the answers point to different " + "provisions (" + "; ".join( + f"{o['answer']} -> {o.get('section') or 'none'}" + for o in applied) + "): not settled") + out["needs"].append(_resolver_need(pairing, r, at, status)) + return out +