| """Score the business PII detector on text from a different generator (the HIPAA test set), 2026-10-29. |
| |
| Its own test set comes from the same generators as its training data, so it scores perfectly; this shows how it does |
| on sentences written another way. Only the types both sets share are counted. |
| python models/cross_test.py |
| """ |
|
|
| import json |
| from pathlib import Path |
|
|
| from transformers import pipeline |
|
|
| ROOT = Path(__file__).resolve().parent |
| SAME = {"NAME": "PERSON", "PHONE": "PHONE", "EMAIL": "EMAIL", "DATE": "DATE", "IP_ADDRESS": "IP_ADDRESS", "URL": "URL"} |
| tagger = pipeline("token-classification", model=str(ROOT / "business-pii-detector/out"), aggregation_strategy="simple", device=0) |
| rows = [json.loads(l) for l in open(ROOT / "hipaa-phi-detector/data/test.jsonl", encoding="utf-8")] |
| tp = fp = fn = 0 |
| for r in rows: |
| gold = {(SAME[e["label"]], e["start"], e["end"]) for e in r["entities"] if e["label"] in SAME} |
| pred = set() |
| for p in tagger(r["text"]): |
| if p["entity_group"] in SAME.values(): |
| s = p["start"] + (len(r["text"][p["start"]:p["end"]]) - len(r["text"][p["start"]:p["end"]].lstrip())) |
| pred.add((p["entity_group"], s, p["end"])) |
| tp += len(gold & pred) |
| fp += len(pred - gold) |
| fn += len(gold - pred) |
| out = {"test": "hipaa-phi-detector test set, shared types only", "rows": len(rows), "precision": round(tp / (tp + fp), 3), |
| "recall": round(tp / (tp + fn), 3), "f1": round(2 * tp / (2 * tp + fp + fn), 3)} |
| print(json.dumps(out)) |
| s = json.loads((ROOT / "business-pii-detector/scores.json").read_text(encoding="utf-8")) |
| s["cross_test"] = out |
| (ROOT / "business-pii-detector/scores.json").write_text(json.dumps(s, indent=2) + "\n", encoding="utf-8") |
|
|
