Add hybrid PII scripts: nergal.py and frozen scrub_pii.py

scrub_pii.py

@@ -0,0 +1,746 @@
+"""Regex PII scrub for Polish web/official text.
+
+Replaces emails, phones, PESEL/NIP/REGON/KRS, land-register (KW) numbers, electronic contact addresses and
+account numbers in place so sentence structure survives. Names of public officials are left untouched —
+that is intentional, not a gap. Phones map to [Telefon]; everything else
+to [PII]. Bare PESEL requires checksum and date validation; explicit identifier
+labels also redact damaged numbers (including common OCR O/I/l substitutions).
+NIP/REGON/KRS and identity documents require explicit nearby labels. KW numbers
+need a nearby KW/księga wieczysta label, or the full XX0X/00000000/0 form with a
+valid check digit. Passport
+variants include one-letter and diplomatic IDs. Foreign IBANs use country
+lengths and MOD-97. ePUAP paths require a label; e-Doreczenia uses its AE:PL form.
+Wrapped email domains, local parts hyphenated across one line break and small
+extraction gaps around @/hyphens are supported.
+Explicit phone extensions and terminal suffix ranges are included; room numbers are not. Labelled numeric
+PINs (including URL pin= values) map to [PII] before phone detection.
+Contact/helpline headings cover consecutive descriptive phone-list entries;
+unrelated lines end the list. Bounded staff/address-directory evidence also covers
+formatted phone fields. Labelled full-number ranges retain shared prefixes.
+Phones require a nearby contact cue, a Polish +48/0048 prefix, or explicit
+international country/trunk notation such as +CC (0). With a cue, the EUR-Lex
+"(32-2) 299 11 11" country-area form counts as international. Short service numbers
+need strong labels; 116xxx numbers also accept nearby telephone prose. Unlabelled
+domestic numbers are left for audit because table cells have the same shapes.
+Flattened tables can glue labels to both neighbours ("Mödlingtel.: … 38112faks:");
+such glued labels count as labels and end the preceding number.
+
+Call after HTML-to-text, before the parquet is written.
+"""
+from __future__ import annotations
+import datetime as dt
+import re
+
+PHONE_TAG = "[Telefon]"
+PII_TAG = "[PII]"
+COUNTS = ("email", "phone", "pesel", "nip", "regon", "account", "document", "krs", "electronic_address", "pin",
+          "land_register")
+
+# Mobile + geographic area codes (2-digit national prefix after trunk 0 / +48).
+_PL_PREFIX = {
+    "12", "13", "14", "15", "16", "17", "18", "22", "23", "24", "25", "26", "29",
+    "32", "33", "34", "39", "41", "42", "43", "44", "45", "46", "47", "48",
+    "50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
+    "60", "61", "62", "63", "65", "66", "67", "68", "69",
+    "70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
+    "80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
+    "91", "94", "95",
+}
+
+_EMAIL_DOMAIN = r"[^\W_](?:[^\W_]|-[ \t]{0,3}(?=[^\W_]))*"
+# PDF extraction can hyphenate a local part across one line break: "jan-\nna.k@example.com".
+# The fragment is capped at the 64-character local-part limit so long tokens stay linear.
+_EMAIL_START = r"\b(?:[\w.%+&-]{0,63}[^\W_]-[ \t]*\r?\n[ \t]*)?[\w.%+&-]+[ \t]{0,3}@"
+_EMAIL_RE = re.compile(_EMAIL_START + r"[ \t]{0,3}(?:" + _EMAIL_DOMAIN + r"\.)+[^\W\d_]{2,}\b")
+_EMAIL_WRAP_RE = re.compile(_EMAIL_START + r"[ \t]{0,3}(?:" + _EMAIL_DOMAIN
+    + r"\.)+[ \t]*\r?\n[ \t]*(?:" + _EMAIL_DOMAIN + r"\.)*[^\W\d_]{2,}\b")
+# Damaged contact fields can retain only the local part and @.
+_EMAIL_FRAGMENT_RE = re.compile(_EMAIL_START + r"(?=[ \t]*\r?$)", re.M)
+_EMAIL_LABEL_RE = re.compile(r"\be[ -]?mail[ \t]*:[ \t]*\Z", re.I)
+_EDELIVERY_RE = re.compile(r"\bAE:PL-\d{5}-\d{5}-[A-Z0-9]{5}-\d{2}(?![\w-])", re.I)
+_EPUAP_RE = re.compile(r"(?<![\w/])/[\w-]+/[\w-]+(?![\w/-])")
+_EPUAP_LABEL_RE = re.compile(
+    r"\b(?:e[ -]?puap|elektroniczna[ \t]+skrzynka[ \t]+podawcza)\b[^\n/\[\]]{0,50}\Z", re.I)
+_PIN_RE = re.compile(
+    r"(?P<label>\bPIN[ \t]*[:=][ \t]*)"
+    r"(?P<number>[\u202a-\u202e\u2066-\u2069]*\d(?:[ \t]?\d){3,15}#?)"
+    r"(?!\w|[ \t]*\d)", re.I)
+# Horizontal Unicode spaces from HTML/PDF extraction; preserve paragraph breaks.
+_SPACES = str.maketrans({c: " " for c in "\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000"})
+# Country lengths checked against Apache Commons Validator's IBAN registry table:
+# https://commons.apache.org/proper/commons-validator/xref/org/apache/commons/validator/routines/IBANValidator.html
+# ponytail: country/length/MOD-97 only; add national BBAN rules if false positives appear.
+_IBAN_LENGTHS = {
+    country: length for length, countries in {
+        15: "NO", 16: "BE", 18: "DK FI AX FK FO GL NL SD",
+        19: "MK SI", 20: "AT BA EE KZ LT LU MN XK",
+        21: "CH HR LI LV", 22: "BG BH CR DE GB IM JE GG GE IE ME RS VA",
+        23: "AE GI IL IQ OM SO TL", 24: "AD CZ ES MD PK RO SA SE SK TN VG",
+        25: "LY PT ST", 26: "IS TR",
+        27: "BI DJ FR GF GP MQ RE PF TF YT NC BL MF PM WF GR IT MC MR SM",
+        28: "AL AZ BY CY DO GT HN HU LB NI PL SV", 29: "BR EG PS QA UA",
+        30: "JO KW MU YE", 31: "MT SC", 32: "LC", 33: "RU",
+    }.items() for country in countries.split()
+}
+_IBAN_RE = re.compile(
+    r"\b(?:" + "|".join(
+        country + r"[ \t-]*[0-9]{2}(?:[ \t-]*[A-Z0-9]){" + str(length - 4) + "}"
+        for country, length in _IBAN_LENGTHS.items()
+    ) + r")\b", re.I,
+)
+# Optional PL, then 26 digits with short space/tab/hyphen gaps (invoice style).
+_ACCOUNT_RE = re.compile(r"\b(?:PL[ \t-]*)?(?:\d[ \t-]*){25}\d\b", re.I)
+_REGON14_RE = re.compile(r"\b\d{14}\b")
+_PESEL_RE = re.compile(r"\b\d{11}\b")
+_NIP_DASH_RE = re.compile(r"\b(?:PL[ \t]*)?\d{3}(?:[- \t]\d{3}[- \t]\d{2}[- \t]\d{2}|[- \t]\d{2}[- \t]\d{2}[- \t]\d{3})\b", re.I)
+_NIP_RE = re.compile(r"\b(?:PL[ \t]*)?\d{10}\b", re.I)
+_DOCUMENT_RE = re.compile(
+    r"\b(?P<label>(?:dow[oó]d(?:u|em)?[ \t]+osobist(?:y|ego|ym)|"
+    r"(?:nr|numer)[ \t]+dowodu|paszport(?:u|em)?(?:[ \t]+dyplomatyczn(?:y|ego|ym))?)"
+    r"[ \t]*(?:(?:seria[ \t]+i[ \t]+numer|serii|seria|numer|nr)\.?[ \t]*)?"
+    r"[:=-]?[ \t]*)"
+    r"(?P<number>[A-Z]{3}[ \t-]?[0-9OIl]{6}|[A-Z]{2}[ \t-]?[0-9OIl]{7}|[A-Z][ \t-]?[0-9]{6,9}|[0-9]{4,12})\b", re.I,
+)
+_LABELLED_ID_RE = re.compile(
+    r"\b(?P<label>(?P<kind>PESEL|NIP|REGON)\b"
+    r"(?:[ \t]+(?:Gminy|Powiatu|Miasta|firmy)(?:[ \t]+[^\W\d_][^\W\d_-]*){0,4})?[ \t]{0,8}"
+    r"(?:(?:nr\.?|numer)[ \t]{0,8})?[:=.\-]?[ \t]{0,8}(?:\r?\n[ \t]{0,8})?)"
+    r"(?P<number>(?:PL[ \t]*)?(?:[0-9OIl]{9,14}|\d(?:[ \t-]?\d){8,13}))"
+    r"(?!\w|[ \t-]*\d)", re.I,
+)
+# Land-register (KW) number: court code / number / check digit, e.g. WA1M/00123456/3.
+_LAND_REGISTER_RE = re.compile(
+    r"\b(?P<court>[A-Z]{2}\d[A-Z])[ \t]*[/ \t][ \t]*(?P<number>\d{1,8})[ \t]*/[ \t]*(?P<check>\d)(?![\w/])")
+_LAND_REGISTER_LABEL_RE = re.compile(
+    r"(?:(?-i:\bKW\b)|\bks(?:\.|i[eęą]\w*)[ \t]+wieczyst\w*)[^\n]{0,30}\Z", re.I)
+# Check-digit values; court codes skip Q and V.
+_LAND_REGISTER_VALUES = {c: i for i, c in enumerate("0123456789XABCDEFGHIJKLMNOPRSTUWYZ")}
+_REGON9_RE = re.compile(r"\b\d{9}\b")
+_ID_LABEL_END = r"[ \t]{0,8}(?:(?:nr\.?|numer)[ \t]{0,8})?[:=.\-]?[ \t]{0,8}(?:\r?\n[ \t]{0,8})?\Z"
+_NIP_LABEL_RE = re.compile(r"\bNIP\b" + _ID_LABEL_END, re.I)
+_REGON_LABEL_RE = re.compile(r"\bREGON\b" + _ID_LABEL_END, re.I)
+_KRS_RE = re.compile(r"\b\d{6,10}(?!\w|[ \t-]*\d)")
+# Abbreviation or written-out register name, optionally "pod nr/numerem".
+_KRS_LABEL_RE = re.compile(
+    r"(?:\bKRS\b|\bKrajow\w*[ \t]+Rejestr\w*[ \t]+S[aą]dow\w*)"
+    r"(?:[ \t]{0,8},?[ \t]{0,8}pod[ \t]{1,8}(?:nr\.?|numerem))?" + _ID_LABEL_END, re.I)
+# Separators are short and local: one newline *or* a few punct/spaces.
+# Letters and blank lines break the match so body text is not swallowed.
+_SEP = r"(?:[ \t./\-()\u2013\u2014]{0,3}|\n)"
+_VANITY_PHONE_RE = re.compile(
+    r"(?<!\w)\+1[ \t-]+\d{3}[ \t-]+(?:\d{3}-[A-Z]{4}|[A-Z]{7})"
+    r"(?:[ \t]+\(\d{4,7}\))?(?!\w)")
+# Flattened tables glue labels to both neighbours: "Mödlingtel.: +43 (0) 512 34-56712faks:".
+# Glued "tel" needs its dot and glued "fax" a non-letter before it ("Hotel:", "Halifax:").
+_GLUED_PHONE_LABEL = r"(?:tel\.|telefon|t[ée]l[ée]phone|t[ée]l[ée]copieur|(?<![^\W\d_])fa(?:x|ks))[ \t]*:"
+# A number ends at a word boundary or where a glued contact label starts.
+_PHONE_END = r"(?:(?!\w)|(?=" + _GLUED_PHONE_LABEL + r"|(?:e-?mail|t[ée]lex)[ \t]*:))"
+_PHONE_RE = re.compile(
+    _VANITY_PHONE_RE.pattern + r"|(?:(?<!\w)|(?<=telefon)|(?<=tel)|(?<=faks)|(?<=fax))(?:\(?\+[ \t]*)?"
+    r"(?:\((?:0|0?\d{2,4}(?:-\d{1,2})?)\)" + _SEP + r")?"
+    r"\d(?:" + _SEP + r"\d){4,}" + _PHONE_END, re.I,
+)
+_SHORT_PHONE_LABEL_RE = re.compile(
+    r"(?:\b(?:tel(?:efon\w*)?(?:(?:\.[ \t]*|[ \t]+)(?:kom[oó]rk\w*|lokaln\w*|stacjonarn\w*))?|fax|faks(?:em)?|phone|"
+    r"toll[ -]free|numer[ \t]+bezpłatny)|[☎☏📞📱])[.: \t/]{0,8}\Z|"
+    r"\binfolini[^\W\d_]*[^\d\n]{0,80}:[ \t]*\Z|" + _GLUED_PHONE_LABEL + r"[ \t]*\Z", re.I)
+_PHONE_LABEL_RE = re.compile(
+    r"\b(?:tel(?:efon[^\W\d_]*)?|fax|faks(?:em)?|kom(?:[oó]rk[^\W\d_]*)?|gsm|"
+    r"infolini[^\W\d_]*|(?:za)?dzwo[ńn][^\W\d_]*|Blikiem|"
+    r"lini[ęaąie][ \t]+wsparcia)(?![^\W\d_])|"
+    r"\bkontakt(?=tel[.:])|[☎☏📞📱]", re.I,
+)
+_PHONE_BREAK_RE = re.compile(
+    r"\n|\.[ \t]+(?=\d)|[ \t]+(?=\(?(?:[01]?\d|2[0-3])[.:][0-5]\d"
+    r"|\d{1,2}[.)][ \t]+\d{1,2}[./]\d{1,2}[./](?:19|20)\d{2})")
+_SECTION_MARKER_RE = re.compile(r"\d{1,2}[.)](?!\d)")
+_PHONE_EXTENSION_RE =re.compile(r"(?:(?:[ \t]+,?[ \t]*|,[ \t]*)wew(?:n(?:ętrzny)?)?\.?[ \t]*\d{1,5}|[ \t]+do[ \t]+\d{1,3}(?=[ \t]*(?:[.;,](?!\d)|\r?\n|\Z)))(?!\w|[ \t]*\d)", re.I)
+_SERVICE_PHONE_RE = re.compile(r"(?<![\w+])[1-9]\d{2}(?![\w\d]|[ \t./()-]*\d)")
+_SERVICE_PHONE_LABEL_RE = re.compile(
+    r"(?:\b(?:tel(?:efon\w*)?|phone)[.: \t]{0,8}(?:alarmow\w*[.: \t]{0,8})?|"
+    r"\b(?:numer[ \t]+)?bezpłatny[.: \t]{0,8}|"
+    r"\b(?:bezpłatny[ \t]+)?numer[ \t]+alarmowy[.: \t]{0,8}|"
+    r"(?:^|\n)[ \t]*(?:Straż pożarna|Policja|Pogotowie|Ratunek|Lekarz pogotowia ratunkowego)"
+    r"(?:[ \t]+\([^()\n]{1,60}\))?[ \t]*:[ \t]*)"
+    r"(?:[1-9]\d{2}[ \t]*(?:[,;]|i|lub|oraz)[ \t]*)*\Z", re.I)
+_OTHER_NUMBER_LABEL_RE = re.compile(r"\b(?:NIP|REGON|PESEL|KRS|ISBN|kod)\b[^\d\n]{0,20}\Z", re.I)
+_PHONE_LIST_HEADING_RE = re.compile(
+    r"[ \t]*(?:numery[ \t]+kontaktowe|telefony(?:[ \t]+(?:kontaktowe|zaufania))?|fax|faks|zapisz[ \t]+się|"
+    r"(?:[^.!?\n]{0,40}\?[ \t]*)?(?:za)?dzwoń!?)"
+    r"(?:[ \t]*\([^()\n]{1,80}\))?[ \t]*:?[ \t]*", re.I)
+_CONTACT_INTRO_RE = re.compile(
+    r"\b(?:kontakt\b|zapisy\b|zapisz[ \t]+się\b|rejestracja\b)"
+    r"(?:nr\.|[^\n.!?;]){0,75}\Z", re.I)
+_PHONE_HEADING_RE = re.compile(
+    r"(?<!\[)\b(?:tel(?:efon[^\W\d_]*)?\b|kontakt\b|zapisy\b|zapisz[ \t]+się\b|"
+    r"(?:za)?dzwo[ńn][^\W\d_]*\b|(?:pod|na)[ \t]+numer(?:em)?)"
+    r"(?:tj\.|nr\.|[^\d\n.!?;\[\]]){0,75}:?[ \t]*(?:\n[ \t]*){1,4}\Z", re.I)
+_CONTACT_EXCLUDE_RE = re.compile(r"\b(?:spraw\w*|kod\w*|kwot\w*|statystyk\w*|taryf\w*)\b", re.I)
+_PHONE_LINE_RE = re.compile(r"(?<!\w)\d{2,3}(?:[ \t-]\d{2,4}){2,3}(?!\w)")
+_STAFF_ROLE_RE = re.compile(
+    r"\b(?:inspektor|referent|koordynator|psycholog|księgowość|księgowy|księgowa|"
+    r"pracownicy[ \t]+socjalni|łowczy|podłowczy)\b", re.I)
+
+
+def _phone_list_context(text: str, start: int) -> bool:
+    # ponytail: 2,000-character lookback and 400-character entry suffix; longer
+    # lists need repeated headings or a section parser. Never cross prose.
+    end = text.find('\n', start, start + 401)
+    if end < 0 and len(text) > start + 400:
+        return False
+    lines = text[max(0, start - 2000):end if end >= 0 else len(text)].splitlines()
+    if start > 2000:
+        lines = lines[1:]  # A truncated line cannot establish a heading.
+    for i, line in enumerate(reversed(lines)):
+        if _PHONE_LIST_HEADING_RE.fullmatch(line):
+            return i > 0
+        if not line.strip():
+            continue
+        phone = _PHONE_RE.search(line)
+        if not phone or not _phone_ok(phone[0], short=True) or _is_amount(line, *phone.span()):
+            return False
+        if i == 0 and phone.start() != start - (text.rfind('\n', 0, start) + 1):
+            return False  # Numbers in an entry's description are not list items.
+        before, after = line[:phone.start()].strip(), line[phone.end():].strip()
+        if _OTHER_NUMBER_LABEL_RE.search(before):
+            return False
+        # Either "City: number" or "number – helpline description". Bare
+        # numeric rows are ambiguous even below an earlier contact heading.
+        named = re.fullmatch(r"[^\W\d_][^\d\n:;]{0,79}[:–—-]", before)
+        described = before in ('', '•', '-', '*') and re.match(r"[–—-][ \t]+[^\W\d_]", after)
+        dialling = re.fullmatch(r"[-•*]?[ \t]*(?:dzwoniąc[ \t]+)?z[ \t]+[^\W\d_][^\d\n:;.!?]{0,100}", before, re.I)
+        if not (((named or dialling) and after in ('', ',', ';', '.')) or described):
+            return False
+    return False
+
+
+def _phone_context(text: str, start: int, raw: str) -> bool:
+    # A previous redaction is a boundary, not a fresh "Telefon" cue. Include
+    # one marker's extra width so the 75-character window cannot bisect it.
+    before = re.split(r"\n[ \t]*\n|\[Telefon\]|\[PII\]",
+                      text[max(0, start - 75 - len(PHONE_TAG)):start])[-1][-75:]
+    if _OTHER_NUMBER_LABEL_RE.search(before):
+        return False
+    intro = _CONTACT_INTRO_RE.search(before)
+    if intro and not _CONTACT_EXCLUDE_RE.search(intro[0]):
+        return True
+    # A contact email must not erase an explicit contact label for the phone
+    # following it. Other redactions still form boundaries.
+    if re.search(r'\bkontakt[ \t]*:[ \t]*\[PII\][ \t,;]*\Z',
+                 text[max(0, start-75):start], re.I):
+        return True
+    lead = re.split(r'\[Telefon\]|\[PII\]', text[max(0, start-240):start])[-1]
+    # "Pod numerem" also introduces contract/case references. Require a
+    # communication cue when the number has no explicit telephone label.
+    number_contact = bool((_PHONE_LABEL_RE.search(lead) or re.search(
+        r'\b(?:informacj[^\W\d_]*|zgłosz[^\W\d_]*|zapis[^\W\d_]*|SMS(?:-a)?|połączeni[^\W\d_]*)\b', lead, re.I))
+        and not _CONTACT_EXCLUDE_RE.search(lead)
+        and not re.search(r'\b(?:umow|faktur|dokument)[^\W\d_]*\b', lead, re.I))
+    heading = _PHONE_HEADING_RE.search(text[max(0, start-160):start])
+    if (heading and not _CONTACT_EXCLUDE_RE.search(heading[0])
+            and (not re.match(r'(?:pod|na)\b', heading[0], re.I)
+                 or _PHONE_LABEL_RE.search(heading[0]) or number_contact)):
+        return True
+    if (number_contact and re.search(r'\b(?:pod[ \t]+numerem|na[ \t]+numer)[ \t]*:?[ \t]*\Z', before, re.I)):
+        return True
+    if re.search(r'\bтелефону[ \t]*:[ \t]*\Z', before, re.I):
+        return True
+    # An international phone may precede a labelled fax in the same contact list.
+    # Keep this cue adjacent: arbitrary later contact prose is not evidence.
+    after = text[start + len(raw):start + len(raw) + 40]
+    if re.match(r'[ \t]*[–—-][ \t]+telefon\b', after, re.I):
+        return True
+    if (re.match(r"\(?(?:\+|00)", raw)
+            and re.match(r"[ \t]*[,;][ \t]*(?:fax|faks(?:em)?)[.: \t]*(?=[+(0-9])", after, re.I)):
+        return True
+    # A foreign-looking +number can also be an increment in a financial table.
+    return bool(re.match(r"\(?(?:\+|00)[ \t]*48|\+\d{1,3}[ \t]+\(0\)", raw)
+                or _PHONE_LABEL_RE.search(before))
+
+
+def _directory_context(text: str, start: int, end: int) -> bool:
+    """Recognize phones in bounded staff and named-address contact records."""
+    raw = text[start:end]
+    foreign = raw.startswith('+') and _phone_ok(raw)
+    if not foreign and (not _pl_national_ok(_digits(raw)) or not re.search(r'[ \t-]', raw)):
+        return False
+    # ponytail: local 1,600-character directory evidence; longer isolated rows
+    # need preserved upstream table structure, not an unbounded document cue.
+    left, right = max(0, start-800), min(len(text), end+800)
+    nearby = text[left:right]
+    line_start, line_end = text.rfind('\n', 0, start)+1, text.find('\n', end)
+    line_end = len(text) if line_end < 0 else line_end
+    before, after = text[line_start:start], text[end:line_end]
+    if _CONTACT_EXCLUDE_RE.search(before) or _OTHER_NUMBER_LABEL_RE.search(before):
+        return False
+    if foreign:
+        # A comma-delimited name/address entry needs a labelled phone nearby;
+        # a +number alone could be an increment in a financial table.
+        address = re.search(r'(?:^|,)[ \t]*[^\W\d_][^,\n]{1,120},'
+                            r'[^\n]{0,160}\d[^\n]{0,80},[ \t]*\Z', before)
+        return bool(address and any(
+            _SHORT_PHONE_LABEL_RE.search(nearby[max(0,m.start()-80):m.start()])
+            for m in _PHONE_RE.finditer(nearby) if m[0].startswith('+') and _phone_ok(m[0])))
+    standalone = not before.strip() and not after.strip()
+    lead = text[max(0, start-400):start]
+    if (standalone and re.search(r'\b(?:infolini[^\W\d_]*|helpline)\b', lead, re.I)
+            and not _CONTACT_EXCLUDE_RE.search(lead)):
+        return True
+    if (standalone and re.search(r'\bnr\.?[ \t]+telefonu\b', nearby, re.I)
+            and re.search(r'\b(?:pok[oó]j|pokoju)\b', nearby, re.I)
+            and re.search(r'\bnazwisko\b', nearby, re.I)):
+        return True
+    # Email/name/role records and phone-led service directories repeat formatted
+    # values. A lone staff title beside a number is insufficient evidence.
+    phones = [m for m in _PHONE_LINE_RE.finditer(nearby)
+              if _pl_national_ok(_digits(m[0])) and not _is_amount(nearby, *m.span())]
+    roles = list(_STAFF_ROLE_RE.finditer(nearby))
+    if len(phones) < 2 or len(roles) < 2:
+        return False
+    return bool((re.search(r'\[PII\]', before[-100:] + after[:100]) and
+                 re.search(r'[^\W\d_]', before[-100:]))
+                or (not before.strip() and re.match(r'[ \t]+[^\W\d_]', after))
+                or _STAFF_ROLE_RE.search(before[-100:]))
+
+
+# Amounts can look like phones or checksum-valid national IDs. Keep the
+# currency adjacent to the number, allowing a decimal, multiplier and line wrap.
+_MONEY_GAP = r"[ \t]*(?:\n[ \t]*)?"
+_CURRENCY = r"(?:PLN|z[łl](?:ot(?:ych|ego|emu|ymi|ym|y|e))?|EUR(?:O)?|USD|GBP|CHF)\b|[%€$£]"
+_MONEY_AFTER_RE = re.compile(
+    r"(?:[,.][0-9]+)?" + _MONEY_GAP
+    + r"(?:(?:tys\.?|mln|mld|bln)\b\.?" + _MONEY_GAP + r")?"
+    + r"(?:" + _CURRENCY + r")", re.I,
+)
+_MONEY_BEFORE_RE = re.compile(r"(?:\b(?:PLN|EUR|USD|GBP|CHF)|[€$£])" + _MONEY_GAP + r"$", re.I)
+
+
+def _is_amount(text: str, start: int, end: int) -> bool:
+    return bool(_MONEY_AFTER_RE.match(text, end) or _MONEY_BEFORE_RE.search(text[max(0, start - 32):start]))
+
+
+def _digits(s: str) -> str:
+    return re.sub(r"\D", "", s)
+
+
+def _pesel_ok(d: str) -> bool:
+    if len(d) != 11 or not d.isdigit():
+        return False
+    weights = (1, 3, 7, 9, 1, 3, 7, 9, 1, 3)
+    check = sum(w * int(x) for w, x in zip(weights, d[:-1]))
+    if str((10 - check % 10) % 10) != d[-1]:
+        return False
+    yy, mm, dd = int(d[0:2]), int(d[2:4]), int(d[4:6])
+    century = {0: 1900, 1: 2000, 2: 2100, 3: 2200, 4: 1800}.get(mm // 20)
+    if century is None:
+        return False
+    try:
+        dt.date(century + yy, mm % 20, dd)
+    except ValueError:
+        return False
+    return True
+
+
+def _nip_ok(d: str) -> bool:
+    if len(d) != 10 or not d.isdigit():
+        return False
+    weights = (6, 5, 7, 2, 3, 4, 5, 6, 7)
+    rem = sum(w * int(x) for w, x in zip(weights, d[:-1])) % 11
+    return rem != 10 and rem == int(d[-1])
+
+
+def _regon_ok(d: str) -> bool:
+    if not d.isdigit() or len(d) not in (9, 14):
+        return False
+    weights = ((8, 9, 2, 3, 4, 5, 6, 7) if len(d) == 9
+               else (2, 4, 8, 5, 0, 9, 7, 3, 6, 1, 2, 4, 8))
+    rem = sum(w * int(x) for w, x in zip(weights, d[:-1])) % 11
+    if rem == 10:
+        rem = 0
+    return rem == int(d[-1])
+
+
+def _land_register_ok(court: str, number: str, check: str) -> bool:
+    code = court + number.zfill(8)
+    if not all(c in _LAND_REGISTER_VALUES for c in code):
+        return False
+    weights = (1, 3, 7) * 4
+    return sum(w * _LAND_REGISTER_VALUES[c] for w, c in zip(weights, code)) % 10 == int(check)
+
+
+def _iban_ok(raw: str) -> bool:
+    # Do not assemble an alphanumeric account from neighbouring prose words.
+    # Compact and ordinary four-character printed groups remain supported.
+    groups = re.split(r'[ \t-]+', raw)
+    if len(groups) > 1 and any(re.search(r'[A-Za-z]{5}', group) for group in groups):
+        return False
+    compact = re.sub(r"[\s-]+", "", raw).upper()
+    if compact.isdigit() and len(compact) == 26:
+        compact = "PL" + compact
+    if not re.fullmatch(r"[A-Z]{2}[0-9]{2}[A-Z0-9]+", compact):
+        return False
+    if len(compact) != _IBAN_LENGTHS.get(compact[:2]) or not 2 <= int(compact[2:4]) <= 98:
+        return False
+    if compact.startswith("PL") and not compact[2:].isdigit():
+        return False
+    rearranged = compact[4:] + compact[:4]
+    nums = "".join(str(ord(c) - 55) if c.isalpha() else c for c in rearranged)
+    return int(nums) % 97 == 1
+
+
+def _tag(start, end, tag):
+    return tag
+
+
+def _replace_epuap(text: str, *, mask=_tag) -> tuple[str, int]:
+    last_end, count = None, 0
+
+    def replace(m):
+        nonlocal last_end, count
+        labelled = _EPUAP_LABEL_RE.search(text, max(0, m.start()-64), m.start())
+        continued = last_end is not None and re.fullmatch(r'[ \t]*[;,][ \t]*|[ \t]+(?:lub|albo|i|oraz)[ \t]+', text[last_end:m.start()], re.I)
+        if not labelled and not continued:
+            return m[0]
+        last_end = m.end()
+        count += 1
+        return mask(m.start(), m.end(), PII_TAG)
+
+    out = _EPUAP_RE.sub(replace, text)
+    return out, count
+
+
+def _replace_documents(text: str, *, mask=_tag) -> tuple[str, int]:
+    n = 0
+
+    def replace(m):
+        nonlocal n
+        number = re.sub(r"[ \t-]", "", m["number"]).upper()
+        passport = m["label"].lower().startswith("paszport")
+        if passport:
+            short_diplomatic = ('dyplomatyczn' in m['label'].lower()
+                                and re.search(r'\b(?:nr|numer)\b', m['label'], re.I)
+                                and re.fullmatch(r'\d{4,5}', number))
+            if not short_diplomatic and not re.fullmatch(r"[A-Z]{2}[0-9OIL]{7}|[A-Z][0-9]{6,9}|[0-9]{6,12}", number):
+                return m[0]
+        elif not re.fullmatch(r"[A-Z]{3}[0-9OIL]{6}", number):
+            return m[0]
+        n += 1
+        return m["label"] + mask(m.start('number'), m.end('number'), PII_TAG)
+
+    # Passport detection is label/format based; no MRZ check digit is present here.
+    return _DOCUMENT_RE.sub(replace, text), n
+
+
+def _replace_land_registers(text: str, *, mask=_tag) -> tuple[str, int]:
+    n = 0
+
+    def replace(m):
+        nonlocal n
+        # A label covers shortened, spaced or mistyped forms, as for labelled
+        # PESEL/NIP. Unlabelled numbers need the full form and a valid check digit.
+        labelled = _LAND_REGISTER_LABEL_RE.search(text, max(0, m.start()-64), m.start())
+        full = re.fullmatch(r"[A-Z]{2}\d[A-Z]/\d{8}/\d", m[0])
+        if not labelled and not (full and _land_register_ok(m['court'], m['number'], m['check'])):
+            return m[0]
+        n += 1
+        return mask(m.start(), m.end(), PII_TAG)
+
+    return _LAND_REGISTER_RE.sub(replace, text), n
+
+
+def _pl_national_ok(d: str) -> bool:
+    return len(d) == 9 and d.isdigit() and d[:2] in _PL_PREFIX
+
+
+def _phone_ok(raw: str, short: bool = False) -> bool:
+    s = raw.strip()
+    if _VANITY_PHONE_RE.fullmatch(s):
+        return short
+    if s.startswith('/') or s.endswith('/'):
+        return False
+    s = re.sub(r'^\(\+', '+', s)
+    s = re.sub(r'^\(00\)[ \t]*', '00', s)
+    if re.fullmatch(r"\d{4}[-./]\d{2}[-./]\d{2}", s):
+        return False
+    if re.fullmatch(r'\(?(?:[01]?\d|2[0-3])[.:][0-5]\d[ \t]*[–—-][ \t]*(?:[01]?\d|2[0-3])[.:][0-5]\d\)?', s):
+        return False
+    if re.fullmatch(r"\d{2}-\d{3}", s):  # postal code
+        return False
+    # Ministry / court file numbers: BPRM.4820.2.3.2020, LUB-OMK.601.1.2024.3.
+    # They never start with "+"; dotted international phones do: +218.21.555.0123.
+    if not s.startswith('+') and (
+            s.count(".") >= 3 or (s.count(".") >= 1 and re.search(r"(?<!\d)20\d{2}(?!\d)", s))):
+        return False
+    if re.search(r"(?<!\d)\d{1,2}[-./]\d{1,2}[-./](?:19|20)\d{2}(?!\d)", s):
+        return False
+    d = _digits(raw)
+    if short and re.fullmatch(r'[2-9]\d{2}-[2-9]\d{2}-\d{4}', s):
+        return True
+    if short and s.startswith('(0)'):
+        return 7 <= len(d) <= 12
+    # ponytail: label + shape + length, not a country numbering-plan validator.
+    if short and re.match(r'^\(0?\d{2,4}(?:-\d{1,2})?\)', s):
+        return 8 <= len(d) <= 15
+    # EUR-Lex puts country and area code in parentheses: "(32-2) 299 11 11" is +32 2 299 11 11.
+    s = re.sub(r'^\(([1-9]\d{1,2})-(\d{1,2})\)', r'+\1 \2', s)
+    if s.startswith(('+', '00')):
+        international = _digits(s.replace('(0)', ''))
+        if s.startswith('00'):
+            international = international[2:]
+        if not international.startswith('48'):
+            # ponytail: explicit prefix + length, not a global numbering-plan
+            # validator. Add country metadata if review finds false positives.
+            return (7 if '(0)' in s else 8) <= len(international) <= 15 and international[0] != '0'
+    if d.startswith("00"):
+        d = d[2:]
+    if d.startswith("48") and len(d) >= 11:
+        rest = d[2:]
+        if rest.startswith("0"):
+            rest = rest[1:]
+        return _pl_national_ok(rest)
+    if d.startswith("0") and len(d) == 11 and d[:2] in {"01", "02", "07"}:
+        return True
+    if d.startswith("0") and len(d) >= 10:
+        return _pl_national_ok(d.lstrip("0")) or (short and len(d) <= 12)
+    return (_pl_national_ok(d)
+            or (short and 5 <= len(d) <= 8 and d[0] != '0')
+            or (short and 9 <= len(d) <= 12 and bool(re.fullmatch(r'\d{2,4}(?:[ \t-]\d{2,4}){2,3}', s)))
+            or (short and 9 <= len(d) <= 12 and bool(re.fullmatch(r'\d{2,4}\.\d{6,8}', s)))
+            or (short and 9 <= len(d) <= 12 and d.startswith('0')))
+
+
+def _replace_checked(text: str, pattern: re.Pattern, tag: str, ok,
+                     label: re.Pattern | None = None, *, mask=_tag) -> tuple[str, int]:
+    n = 0
+
+    def _sub(m):
+        nonlocal n
+        if label is not None and not label.search(text, max(0, m.start() - 64), m.start()):
+            return m.group(0)
+        if pattern is _PESEL_RE and re.search(r'[/?&][^\s]*\Z', text[max(0, m.start()-500):m.start()]):
+            return m.group(0)  # Bare URL path/query digits are not personal identifiers.
+        if not ok(m.group(0)) or (pattern not in (_EMAIL_RE, _EMAIL_WRAP_RE, _EMAIL_FRAGMENT_RE)
+                                 and _is_amount(text, m.start(), m.end())):
+            return m.group(0)
+        n += 1
+        return mask(m.start(), m.end(), tag)
+
+    return pattern.sub(_sub, text), n
+
+
+def _replace_phones(text: str, *, mask=_tag) -> tuple[str, int]:
+    n = 0
+    out = []
+    pos = 0
+    last_phone_end = None
+    last_phone_complete = False
+    while True:
+        m = _PHONE_RE.search(text, pos)
+        if not m:
+            out.append(text[pos:])
+            break
+        raw = m[0]
+        end = m.end()
+        replacement = raw
+        line_end = text.find('\n', m.start(), end)
+        if line_end >= 0 and re.fullmatch(r'\d{1,3}', text[m.start():line_end]):
+            next_end = text.find('\n', line_end+1, end)
+            if _directory_context(text, line_end+1, next_end if next_end >= 0 else end):
+                # A flattened table's room cell is not a phone country prefix.
+                out.append(text[pos:line_end+1])
+                pos = line_end+1
+                continue
+        short = bool(_SHORT_PHONE_LABEL_RE.search(text[max(0, m.start()-100):m.start()]))
+        short = short or (re.fullmatch(r'116[ \t]?\d{3}', raw)
+                          and _phone_context(text, m.start(), raw))
+        short = short or _phone_list_context(text, m.start())
+        # A bare five-digit continuation followed by a place/name can be a
+        # postal address. It needs its own phone label to override that ambiguity.
+        postal = re.fullmatch(r'\d{5}', raw) and re.match(r'[ \t]+[^\W\d_]', text[end:])
+        # So is a 5-6 digit number after a complete phone and a bare line break: a table
+        # cell, "Fax (32-2) 287 25 24\n28 648,00". After a comma a list goes on,
+        # "95-1-511098,\n514262"; after a fragment the next line is its wrapped rest,
+        # "tel. +48 609 \n953 709", or the next local number, "(30) 25 10 22 33 25,\n22 33 28".
+        cell = (last_phone_complete and 5 <= len(_digits(raw)) <= 6
+                and re.fullmatch(r'[ \t]*\n[ \t\n]*', text[last_phone_end:m.start()]))
+        if (last_phone_end is not None and not postal and not cell
+                and re.fullmatch(r"[ \t\n,;]{1,12}|[ \t]+(?:lub|albo)[ \t]+", text[last_phone_end:m.start()], re.I)):
+            short = True
+        if (last_phone_end is not None
+                and re.fullmatch(r'(?:[ \t]+(?:w sprawie\b|\((?:w godzinach|dostępny)\b)'
+                                 r'|[ \t]*[–—-][ \t]+)[^\d\n.!?]{1,200}\n[ \t\n]{0,8}',
+                                 text[last_phone_end:m.start()], re.I)
+                and _pl_national_ok(_digits(raw))):
+            short = True
+        if (short or _phone_context(text, m.start(), raw)
+                or _directory_context(text, m.start(), line_end if line_end >= 0 else end)):
+            if (_is_amount(text, m.start(), end)
+                    and re.fullmatch(r'\d{1,3}(?:\.[ \t]*\d{3})+', raw)):
+                out.extend((text[pos:m.start()], raw))
+                pos = end
+                continue
+            # Stop before a new line or opening hours, but only after a complete
+            # phone. Look past the greedy match's last digit to recognize HH:MM.
+            for gap in _PHONE_BREAK_RE.finditer(text, m.start(), end + 6):
+                if gap.start() >= end:
+                    break
+                # Foreign lengths vary: a valid-looking prefix may still be an
+                # incomplete wrapped number. Only a section marker ends it.
+                if (raw.startswith(('+', '00')) and not re.match(r'(?:\+|00)[ \t]*48', raw)
+                        and gap[0] == '\n'
+                        and not _SECTION_MARKER_RE.match(text, gap.end())):
+                    continue
+                prefix = text[m.start():gap.start()].rstrip(". \t")
+                # A short number also ends at a section marker: "albo 1234567\n2. Szkoła".
+                section = gap[0] == '\n' and bool(_SECTION_MARKER_RE.match(text, gap.end()))
+                if _phone_ok(prefix, short and section) and not _is_amount(text, m.start(), gap.start()):
+                    end = m.start() + len(prefix)
+                    raw = prefix
+                    replacement = raw
+                    break
+            if _is_amount(text, m.start(), end):
+                out.extend((text[pos:m.start()], raw))
+                pos = end
+                continue
+            # A suffix range repeats the final extension digits, not a second
+            # complete phone. Only accept it after a valid full base number.
+            extension = re.fullmatch(r"(.+?)[/-](\d{1,3})", raw)
+            parenthesized_extension = re.fullmatch(r'(.+?)[ \t]+\(\d{1,5}', raw)
+            if not (text[end:end+1] == ')' and parenthesized_extension
+                    and _phone_ok(parenthesized_extension[1])):
+                parenthesized_extension = None
+            pair = re.fullmatch(r"(.+?)[ \t]+[–—-][ \t]+(.+)", raw)
+            shared = re.match(r"\(\d{2,4}\)[ \t]*", pair[1]) if pair else None
+            full_range = (pair and _phone_ok(pair[1], short)
+                          and _phone_ok((shared[0] if shared else '') + pair[2], short))
+            if (_phone_ok(raw, short) or (extension and _phone_ok(extension[1]))
+                    or parenthesized_extension or full_range):
+                if raw.count('(') > raw.count(')') and text[end:end+1] == ')':
+                    end += 1
+                suffix = _PHONE_EXTENSION_RE.match(text, end)
+                if suffix and not _is_amount(text, suffix.start(), suffix.end()):
+                    end = suffix.end()
+                start = m.start()
+                # Parentheses enclosing prose are punctuation, not phone syntax.
+                if raw.startswith('(+') and raw.count('(') > raw.count(')') and text[end-1:end] != ')':
+                    start += 1
+                replacement = text[m.start():start] + mask(start, end, PHONE_TAG)
+                n += 1
+            # ponytail: split only pairs (<=22 digits); longer lists need label
+            # context to distinguish them from accounts. Never redact ID tails.
+            elif 18 <= len(_digits(raw)) <= 22 and "." not in raw:
+                # A slash separates alternatives, the second without the shared
+                # prefix: "(31-30) 274 44 13/274 44 01". Try slashes first.
+                gaps = sorted(re.finditer(r"[ \t]*/[ \t]*|[ \t\n]+", raw), key=lambda g: '/' not in g[0])
+                for gap in gaps:
+                    if _phone_ok(raw[:gap.start()]) and _phone_ok(raw[gap.end():], '/' in gap[0]):
+                        replacement = (mask(m.start(), m.start()+gap.start(), PHONE_TAG)
+                                       + gap[0] + mask(m.start()+gap.end(), end, PHONE_TAG))
+                        n += 2
+                        break
+        out.extend((text[pos:m.start()], replacement))
+        if replacement != raw:
+            last_phone_end = end
+            last_phone_complete = _phone_ok(raw)
+        pos = end
+    return "".join(out), n
+
+
+def _replace_extensions(text: str, *, mask=_tag) -> tuple[str, int]:
+    count = 0
+
+    def replace(m):
+        nonlocal count
+        before = text[max(0, m.start()-500):m.start()]
+        # Only a telephone followed by extension/name entries establishes scope.
+        heading = re.search(r'\btel(?:efon)?[.: \t]*\[Telefon\]'
+                            r'(?P<entries>(?:\s*wewn?\.?[ \t]+\d{1,5}[ \t]+[^\d\n]+)*\s*)\Z', before, re.I)
+        if not heading or _is_amount(text, m.start('number'), m.end('number')):
+            return m[0]
+        count += 1
+        return m['label'] + mask(m.start('number'), m.end('number'), PHONE_TAG)
+
+    output = re.sub(r'(?P<label>(?:^|\n)[ \t]*wewn?\.?[ \t]+)(?P<number>\d{1,5})(?!\w)',
+                    replace, text, flags=re.I)
+    return output, count
+
+
+def scrub_pii(text: str, *, spans: list | None = None) -> tuple[str, dict[str, int]]:
+    """Return (scrubbed_text, counts); optionally append exact original spans.
+
+    Offset tracking is opt-in; corpus ingestion keeps its existing return type
+    and does not allocate a character map. Unicode offsets count code points.
+    """
+    counts = {k: 0 for k in COUNTS}
+    if not text:
+        return text, counts
+    original = text
+    text = text.translate(_SPACES)  # One character per character: offsets survive.
+    offsets = list(range(len(text))) if spans is not None else None
+    edits = []
+
+    def mask(start, end, tag):
+        if spans is not None:
+            a, b = offsets[start], offsets[end-1] + 1
+            spans.append(dict(start=a, end=b, text=original[a:b],
+                              label='phone' if tag == PHONE_TAG else 'pii'))
+            edits.append((start, end, tag))
+        return tag
+
+    def apply(replacer, *args):
+        nonlocal text
+        text, result = replacer(text, *args, mask=mask)
+        # Each replacement pass uses one coordinate system. Apply its edits
+        # backwards only after all regex callbacks have finished.
+        for start, end, tag in reversed(edits):
+            offsets[start:end] = [offsets[start]] * len(tag)
+        edits.clear()
+        return result
+
+    # Longest / most specific first so a 26-digit account is not sliced
+    # into REGON / PESEL / NIP / phone. Checksums live in the replace callback.
+    def pins(value, *, mask):
+        def replace(m):
+            if _is_amount(value, m.start('number'), m.end('number')):
+                return m[0]
+            counts['pin'] += 1
+            return m['label'] + mask(m.start('number'), m.end('number'), PII_TAG)
+        return _PIN_RE.sub(replace, value), None
+    apply(pins)
+    foreign = apply(_replace_checked, _IBAN_RE, PII_TAG, _iban_ok)
+    bare = apply(_replace_checked, _ACCOUNT_RE, PII_TAG, _iban_ok)
+    counts["account"] = foreign + bare
+    counts['electronic_address'] = apply(_replace_checked, _EDELIVERY_RE, PII_TAG, lambda _: True)
+    counts['electronic_address'] += apply(_replace_epuap)
+    counts['krs'] = apply(_replace_checked, _KRS_RE, PII_TAG, lambda _: True, _KRS_LABEL_RE)
+    counts["document"] = apply(_replace_documents)
+    counts["land_register"] = apply(_replace_land_registers)
+    def labelled_ids(value, *, mask):
+        def replace(m):
+            if len(_digits(m['number'])) < 7 or _is_amount(value, m.start('number'), m.end()):
+                return m[0]
+            counts[m['kind'].lower()] += 1
+            return m['label'] + mask(m.start('number'), m.end('number'), PII_TAG)
+        return _LABELLED_ID_RE.sub(replace, value), None
+    apply(labelled_ids)
+    n14 = apply(_replace_checked, _REGON14_RE, PII_TAG, _regon_ok, _REGON_LABEL_RE)
+    n9 = apply(_replace_checked, _REGON9_RE, PII_TAG, _regon_ok, _REGON_LABEL_RE)
+    counts["regon"] += n14 + n9
+    n_dash = apply(_replace_checked, _NIP_DASH_RE, PII_TAG, lambda s: _nip_ok(_digits(s)), _NIP_LABEL_RE)
+    n_plain = apply(_replace_checked, _NIP_RE, PII_TAG, lambda s: _nip_ok(_digits(s)), _NIP_LABEL_RE)
+    counts["nip"] += n_dash + n_plain
+    # The domain break is the last newline; a hyphenated local part may add an earlier one.
+    counts["email"] = apply(_replace_checked, _EMAIL_WRAP_RE, PII_TAG,
+        lambda s: not _EMAIL_RE.fullmatch(s[:s.rindex('\n')].rstrip('. \t\r')))
+    counts["email"] += apply(_replace_checked, _EMAIL_RE, PII_TAG, lambda _: True)
+    counts["email"] += apply(_replace_checked, _EMAIL_FRAGMENT_RE, PII_TAG,
+                             lambda _: True, _EMAIL_LABEL_RE)
+    counts["phone"] = apply(_replace_phones)
+    counts["phone"] += apply(_replace_checked, _SERVICE_PHONE_RE, PHONE_TAG,
+                             lambda _: True, _SERVICE_PHONE_LABEL_RE)
+    counts['phone'] += apply(_replace_extensions)
+    # Contact context takes precedence over coincidental PESEL checksums in
+    # foreign phone numbers; explicitly labelled identifiers were handled first.
+    bare_pesel = apply(_replace_checked, _PESEL_RE, PII_TAG, _pesel_ok)
+    counts['pesel'] += bare_pesel
+    return text, counts