source/cross_test.py

"""Score the business PII detector on text from a different generator (the HIPAA test set), 2026-10-29.

Its own test set comes from the same generators as its training data, so it scores perfectly; this shows how it does
on sentences written another way. Only the types both sets share are counted.
    python models/cross_test.py
"""

import json
from pathlib import Path

from transformers import pipeline

ROOT = Path(__file__).resolve().parent
SAME = {"NAME": "PERSON", "PHONE": "PHONE", "EMAIL": "EMAIL", "DATE": "DATE", "IP_ADDRESS": "IP_ADDRESS", "URL": "URL"}
tagger = pipeline("token-classification", model=str(ROOT / "business-pii-detector/out"), aggregation_strategy="simple", device=0)
rows = [json.loads(l) for l in open(ROOT / "hipaa-phi-detector/data/test.jsonl", encoding="utf-8")]
tp = fp = fn = 0
for r in rows:
    gold = {(SAME[e["label"]], e["start"], e["end"]) for e in r["entities"] if e["label"] in SAME}
    pred = set()
    for p in tagger(r["text"]):
        if p["entity_group"] in SAME.values():
            s = p["start"] + (len(r["text"][p["start"]:p["end"]]) - len(r["text"][p["start"]:p["end"]].lstrip()))
            pred.add((p["entity_group"], s, p["end"]))
    tp += len(gold & pred)
    fp += len(pred - gold)
    fn += len(gold - pred)
out = {"test": "hipaa-phi-detector test set, shared types only", "rows": len(rows), "precision": round(tp / (tp + fp), 3),
       "recall": round(tp / (tp + fn), 3), "f1": round(2 * tp / (2 * tp + fp + fn), 3)}
print(json.dumps(out))
s = json.loads((ROOT / "business-pii-detector/scores.json").read_text(encoding="utf-8"))
s["cross_test"] = out
(ROOT / "business-pii-detector/scores.json").write_text(json.dumps(s, indent=2) + "\n", encoding="utf-8")