Update recognizers.py

recognizers.py

@@ -4,58 +4,119 @@ from presidio_analyzer.nlp_engine import NlpEngineProvider
 from utils import ensure_spacy_model
 
 # ---------------------------------------------------------------------------
-# Spacy model
+# Spacy model (per tokenizzazione) – l'NER vero è sul transformer
 # ---------------------------------------------------------------------------
 ensure_spacy_model("it_core_news_lg")
 
+# ---------------------------------------------------------------------------
+# NLP engine IBRIDO
+#   • spacy        → tokenizzazione, sentence splitting
+#   • transformer  → NER (PERSON, ORG, LOC, DATE) context-aware
+# ---------------------------------------------------------------------------
 _nlp_config = {
-    "nlp_engine_name": "spacy",
-    "models": [{"lang_code": "it", "model_name": "it_core_news_lg"}],
+    "nlp_engine_name": "transformers",
+    "models": [{
+        "lang_code": "it",
+        "model_name": {
+            "spacy":        "it_core_news_lg",
+            "transformers": "Babelscape/wikineural-multilingual-ner",
+        },
+    }],
+    "ner_model_configuration": {
+        "model_to_presidio_entity_mapping": {
+            "PER":  "PERSON",
+            "ORG":  "ORGANIZATION",
+            "LOC":  "LOCATION",
+            "MISC": "NRP",
+        },
+        "low_confidence_score_multiplier": 0.4,
+        "low_score_entity_names": [],
+    },
 }
 
 _nlp_engine = NlpEngineProvider(nlp_configuration=_nlp_config).create_engine()
 analyzer = AnalyzerEngine(nlp_engine=_nlp_engine, supported_languages=["it"])
 
 # ---------------------------------------------------------------------------
-# Custom recognizers – gare d'appalto
+# Helper per recognizer regex
 # ---------------------------------------------------------------------------
-_CUSTOM_RECOGNIZERS: list[PatternRecognizer] = [
-
-    PatternRecognizer(
-        supported_entity="CIG",
+def _r(entity: str, name: str, regex: str, score: float, context: list[str]) -> PatternRecognizer:
+    return PatternRecognizer(
+        supported_entity=entity,
         supported_language="it",
-        patterns=[Pattern("cig", r"\b[0-9A-Z]{10}\b", score=0.6)],
-        context=["CIG", "codice identificativo", "gara", "appalto"],
-    ),
+        patterns=[Pattern(name, regex, score)],
+        context=context,
+    )
 
-    PatternRecognizer(
-        supported_entity="CUP",
-        supported_language="it",
-        patterns=[Pattern("cup", r"\b[A-Z][0-9A-Z]{14}\b", score=0.6)],
-        context=["CUP", "codice unico", "progetto", "investimento"],
-    ),
+# ---------------------------------------------------------------------------
+# Recognizer regex – pattern deterministici italiani
+# (NER transformer copre già PERSON / ORG / LOCATION / DATE)
+# ---------------------------------------------------------------------------
+_RECOGNIZERS: list[PatternRecognizer] = [
 
-    PatternRecognizer(
-        supported_entity="PARTITA_IVA",
-        supported_language="it",
-        patterns=[Pattern("piva", r"\b(IT)?\d{11}\b", score=0.75)],
-        context=["partita IVA", "P.IVA", "PIVA", "IVA", "impresa", "ditta"],
-    ),
+    # ── Identificativi fiscali / societari ────────────────────────────────
 
-    PatternRecognizer(
-        supported_entity="CODICE_FISCALE",
-        supported_language="it",
-        patterns=[Pattern("cf", r"\b[A-Z]{6}\d{2}[A-Z]\d{2}[A-Z]\d{3}[A-Z]\b", score=0.9)],
-        context=["codice fiscale", "C.F.", "CF"],
-    ),
+    _r("PARTITA_IVA", "piva",
+       r"\b(IT)?\d{11}\b", 0.7,
+       ["P.IVA", "partita IVA", "PIVA", "IVA", "impresa", "ditta", "stazione appaltante"]),
 
-    PatternRecognizer(
-        supported_entity="PEC",
-        supported_language="it",
-        patterns=[Pattern("pec", r"\b[\w.\-]+@[\w.\-]*pec[\w.\-]*\.\w+\b", score=0.85)],
-        context=["PEC", "posta certificata", "posta elettronica certificata"],
-    ),
+    _r("CODICE_FISCALE", "cf_persona",
+       r"\b[A-Z]{6}\d{2}[A-Z]\d{2}[A-Z]\d{3}[A-Z]\b", 0.9,
+       ["C.F.", "codice fiscale", "CF", "dipendente", "responsabile"]),
+
+    _r("CODICE_FISCALE", "cf_azienda",
+       r"\b\d{11}\b", 0.65,
+       ["C.F.", "codice fiscale"]),
+
+    _r("REA", "rea",
+       r"\b[A-Z]{2}-\d{4,7}\b", 0.85,
+       ["REA", "registro imprese"]),
+
+    # ── Dati gara ─────────────────────────────────────────────────────────
+
+    _r("CIG", "cig",
+       r"\b[0-9A-Z]{10}\b", 0.6,
+       ["CIG", "codice identificativo", "gara", "appalto"]),
+
+    _r("CUP", "cup",
+       r"\b[A-Z][0-9A-Z]{14}\b", 0.6,
+       ["CUP", "codice unico", "progetto", "investimento"]),
+
+    _r("NUMERO_GARA_ANAC", "num_gara",
+       r"\b\d{7,8}\b", 0.6,
+       ["ANAC", "numero gara", "gara ANAC"]),
+
+    _r("IMPORTO_GARA", "importo",
+       r"€\s*[\d\.]+,\d{2}", 0.75,
+       ["importo", "base d'asta", "aggiudicato", "offerta", "valore"]),
+
+    # ── Comunicazioni ─────���───────────────────────────────────────────────
+
+    _r("PEC", "pec",
+       r"\b[\w.\-]+@[\w.\-]*(?:pec|postacert|legalmail|pecmail)[\w.\-]*\.\w+\b", 0.85,
+       ["PEC", "posta certificata", "posta elettronica certificata"]),
+
+    # ── Persona con titolo professionale (rinforzo NER) ───────────────────
+
+    _r("PERSONA_CON_TITOLO", "titolo_nome",
+       r"\b(Ing\.|Dott\.ssa|Dott\.|Avv\.|Geom\.|Arch\.|Rag\.|Prof\.|Dr\.|Sig\.ra|Sig\.)\s+"
+       r"[A-ZÀ-Ü][a-zà-ü]+(?:\s+[A-ZÀ-Ü][a-zà-ü]+)+", 0.88,
+       ["rappresentante", "responsabile", "referente", "legale", "direttore"]),
+
+    # ── Dati previdenziali / assicurativi / bancari ───────────────────────
+
+    _r("MATRICOLA_INPS", "inps",
+       r"\b\d{10}\b", 0.65,
+       ["INPS", "matricola", "previdenziale", "matricola INPS"]),
+
+    _r("POLIZZA_ASSICURATIVA", "polizza",
+       r"\b[A-Z]{2,5}-\d{4}-[A-Z]{2}-\d{6,12}\b", 0.85,
+       ["polizza", "assicurativa", "N. polizza", "numero polizza"]),
+
+    _r("BIC_SWIFT", "bic",
+       r"\b[A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b", 0.8,
+       ["BIC", "SWIFT", "codice banca", "codice swift"]),
 ]
 
-for _r in _CUSTOM_RECOGNIZERS:
-    analyzer.registry.add_recognizer(_r)
+for _rec in _RECOGNIZERS:
+    analyzer.registry.add_recognizer(_rec)