2.0.0: [PHONE] replaces [Telefon]; [Telefon] stays a rules boundary

hybrid.json

@@ -6,7 +6,7 @@
   "epoch": 5,
   "seed": 202609160,
   "threshold": 0.95,
-  "rules_sha256": "b238d5b88aa3f3d55a24bb051ec93f9179dfb14b2c650441c8e0acb225d81d59",
+  "rules_sha256": "d18662434b3bab69d74e7f63f76a646122cd9d7420de1d70914aaeb903ff2357",
   "eval": {
     "split": "841-dev",
     "gold_amendments": [

nergal.py

@@ -23,7 +23,7 @@ GAP_IDS = [250002, 250003]
 BIO_LABELS = ['O', 'B-phone', 'I-phone', 'B-pii', 'I-pii']
 LABELS = ['phone', 'pii']
 THRESHOLD = 0.95
-RULES_SHA = 'b238d5b88aa3f3d55a24bb051ec93f9179dfb14b2c650441c8e0acb225d81d59'
+RULES_SHA = 'd18662434b3bab69d74e7f63f76a646122cd9d7420de1d70914aaeb903ff2357'
 DTYPES = ('float32', 'float16')
 
 
@@ -168,7 +168,7 @@ class Encoding:
 
 
 def spans_from(module, text):
-    """Rule spans on the original text. Existing [PII]/[Telefon] placeholders do not switch the rules off."""
+    """Rule spans on the original text. Existing [PHONE]/[PII]/[Telefon] placeholders do not switch the rules off."""
     result = []
     module.scrub_pii(text, spans=result)
     return sorted(({k: s[k] for k in ('start', 'end', 'label')} | {'score': 1.0} for s in result),

scrub_pii.py

@@ -2,7 +2,7 @@
 
 Replaces emails, phones, PESEL/NIP/REGON/KRS, land-register (KW) numbers, electronic contact addresses and
 account numbers in place so sentence structure survives. Names of public officials are left untouched —
-that is intentional, not a gap. Phones map to [Telefon]; everything else
+that is intentional, not a gap. Phones map to [PHONE]; everything else
 to [PII]. Bare PESEL requires checksum and date validation; explicit identifier
 labels also redact damaged numbers (including common OCR O/I/l substitutions).
 NIP/REGON/KRS and identity documents require explicit nearby labels. KW numbers
@@ -27,7 +27,7 @@ Phones require a nearby contact cue, a Polish +48/0048 prefix, or explicit
 international country/trunk notation such as +CC (0). With a cue, the EUR-Lex
 "(32-2) 299 11 11" country-area form counts as international. Strong labels also
 admit one-digit country codes and wider hyphenated area codes. Phone policy v3: each
-number of at least 7 digits (keypad letters count) is its own [Telefon], connectors
+number of at least 7 digits (keypad letters count) is its own [PHONE], connectors
 between numbers stay unmasked, and a shorter part (an extension, "/90") stays inside
 its number's span. A phone match with no such number (a short service or emergency
 number, a lone extension) is left as text; a line break alone never splits a
@@ -43,8 +43,11 @@ from __future__ import annotations
 import datetime as dt
 import re
 
-PHONE_TAG = "[Telefon]"
+PHONE_TAG = "[PHONE]"
 PII_TAG = "[PII]"
+LEGACY_PHONE_TAG = "[Telefon]"  # NERGAL 1.x output; still a boundary and a heading in text scrubbed before 2.0
+_BOUNDARY = "|".join(map(re.escape, (PHONE_TAG, PII_TAG, LEGACY_PHONE_TAG)))
+_BOUNDARY_WIDTH = max(map(len, (PHONE_TAG, PII_TAG, LEGACY_PHONE_TAG)))  # 9: same window as 1.x
 COUNTS = ("email", "phone", "pesel", "nip", "regon", "account", "document", "krs", "electronic_address", "pin",
           "land_register", "registry")
 
@@ -248,8 +251,8 @@ def _phone_list_context(text: str, start: int) -> bool:
 def _phone_context(text: str, start: int, raw: str) -> bool:
     # A previous redaction is a boundary, not a fresh "Telefon" cue. Include
     # one marker's extra width so the 75-character window cannot bisect it.
-    before = re.split(r"\n[ \t]*\n|\[Telefon\]|\[PII\]",
-                      text[max(0, start - 75 - len(PHONE_TAG)):start])[-1][-75:]
+    before = re.split(r"\n[ \t]*\n|" + _BOUNDARY,
+                      text[max(0, start - 75 - _BOUNDARY_WIDTH):start])[-1][-75:]
     if _OTHER_NUMBER_LABEL_RE.search(before):
         return False
     intro = _CONTACT_INTRO_RE.search(before)
@@ -260,7 +263,7 @@ def _phone_context(text: str, start: int, raw: str) -> bool:
     if re.search(r'\bkontakt[ \t]*:[ \t]*\[PII\][ \t,;]*\Z',
                  text[max(0, start-75):start], re.I):
         return True
-    lead = re.split(r'\[Telefon\]|\[PII\]', text[max(0, start-240):start])[-1]
+    lead = re.split(_BOUNDARY, text[max(0, start-240):start])[-1]
     # "Pod numerem" also introduces contract/case references. Require a
     # communication cue when the number has no explicit telephone label.
     number_contact = bool((_PHONE_LABEL_RE.search(lead) or re.search(

test_nergal.py

@@ -5,7 +5,7 @@ import unittest
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parent
-RULES_SHA = 'b238d5b88aa3f3d55a24bb051ec93f9179dfb14b2c650441c8e0acb225d81d59'
+RULES_SHA = 'd18662434b3bab69d74e7f63f76a646122cd9d7420de1d70914aaeb903ff2357'
 
 
 class NergalTests(unittest.TestCase):
@@ -58,10 +58,28 @@ class NergalTests(unittest.TestCase):
 
     def test_existing_placeholders_do_not_switch_the_rules_off(self):
         from nergal import rules
-        text = 'Kontakt [Telefon], NIP 1234567802.'  # invented, checksum-valid
-        [span] = rules(text)
-        self.assertEqual(text[span['start']:span['end']], '1234567802')
-        self.assertEqual(rules('a [PII] b [Telefon] c'), [])
+        for marker in ('[PHONE]', '[Telefon]', '[PII]', '[PERSON]'):
+            with self.subTest(marker=marker):
+                text = f'Kontakt {marker}, NIP 1234567802.'  # invented, checksum-valid
+                [span] = rules(text)
+                self.assertEqual(text[span['start']:span['end']], '1234567802')
+        self.assertEqual(rules('a [PII] b [PHONE] c [PERSON] d [Telefon] e'), [])
+
+    def test_phones_are_tagged_phone(self):
+        from nergal import apply_union, rules
+        text = 'Biuro: (22) 123 45 67.'
+        masked = apply_union(text, rules(text))[0]
+        self.assertEqual(masked, 'Biuro: [PHONE].')
+
+    def test_new_and_legacy_phone_tags_are_the_same_boundary(self):
+        from nergal import rules
+        def found(text):  # values, not offsets: the two tags differ in length
+            return [(text[s['start']:s['end']], s['label']) for s in rules(text)]
+        for text in ('Telefon: {} lub 601234567', 'Kontakt: {}, 601234567',  # plain 9 digits: cue-gated
+                     'tel. {}\nwew. 123 Jan Nowak\nwew. 456 sekretariat',
+                     'Kontakt {}: e-mail biuro@example.pl, 601 234 567'):  # invented
+            with self.subTest(text=text):
+                self.assertEqual(found(text.format('[Telefon]')), found(text.format('[PHONE]')))
 
     def test_grouped_national_phones_mask_without_a_cue(self):
         from nergal import rules
@@ -97,7 +115,7 @@ class NergalTests(unittest.TestCase):
             {'start': 20, 'end': 25, 'label': 'pii', 'score': 0.97},
         ]
         masked, counts = scrub_spans(text, rules, model, threshold=0.95)
-        self.assertIn('[Telefon]', masked)
+        self.assertIn('[PHONE]', masked)
         self.assertIn('[PII]', masked)
         self.assertGreater(counts['union_placeholder_chars'], counts['rules_placeholder_chars'])
         self.assertEqual(counts['model_extra_spans'], 1)