| """Build the training and test files for the five Hugging Face models (2026-10-29). |
| |
| Every row is invented or comes from our own published datasets in ../datasets. Nothing is from a real person. |
| python models/build.py writes models/<name>/data/{train,test}.jsonl |
| Test rows for the generated sets come from templates the model never sees in training, so the scores are honest |
| about new sentence shapes, not just new names. |
| """ |
|
|
| import json |
| import random |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parent |
| DS = ROOT.parent / "datasets" |
| rng = random.Random(20261029) |
|
|
|
|
| def read(path): |
| with open(path, encoding="utf-8") as f: |
| return [json.loads(line) for line in f if line.strip()] |
|
|
|
|
| def write(name, split, rows): |
| out = ROOT / name / "data" |
| out.mkdir(parents=True, exist_ok=True) |
| with open(out / f"{split}.jsonl", "w", encoding="utf-8", newline="\n") as f: |
| for r in rows: |
| f.write(json.dumps(r, ensure_ascii=False) + "\n") |
| print(f"{name}/{split}: {len(rows)} rows") |
|
|
|
|
| |
|
|
| PAIR_MAP = { |
| "PERSON_CUSTOMER": "PERSON", "PERSON_STAFF": "PERSON", "ORG_CLIENT": "ORG", "EMAIL": "EMAIL", "PHONE": "PHONE", |
| "STREET_ADDRESS": "ADDRESS", "DATE": "DATE", "MONEY": "MONEY", "ORDER_ID": "ID_NUMBER", |
| } |
|
|
|
|
| def pii(): |
| pairs = read(DS / "synthetic-deidentification-pairs/data/train.jsonl") |
| rows = [{"text": p["text"], "entities": [{"label": PAIR_MAP[e["type"]], "start": e["start"], "end": e["end"]} |
| for e in p["entities"]]} for p in pairs] |
| rng.shuffle(rows) |
| hard_train = read(DS / "pii-span-benchmark-hard/data/train.jsonl") |
| hard_test = read(DS / "pii-span-benchmark-hard/data/test.jsonl") |
| strip = lambda r: {"text": r["text"], "entities": [{"label": e["label"], "start": e["start"], "end": e["end"]} |
| for e in r["entities"]]} |
| write("business-pii-detector", "train", rows[:540] + [strip(r) for r in hard_train]) |
| write("business-pii-detector", "test", rows[540:] + [strip(r) for r in hard_test]) |
|
|
|
|
| |
|
|
| FIRST = ["Marisol", "Dennick", "Aurelio", "Tamsin", "Okonkwo", "Liesel", "Bram", "Yusra", "Corwin", "Ilse", "Nadim", |
| "Petra", "Ravindra", "Sunniva", "Teodor", "Wren", "Zaida", "Hollis", "Imogen", "Kasimir", "Delphine", "Ezio"] |
| LAST = ["Quarrington", "Velasquez-Holt", "Abernethy", "Okafor", "Lindqvist", "Marchetti", "Pemberton", "Szabo", |
| "Thistlewood", "Hakimi", "Brannigan", "Castellanos", "Duchesne", "Fairweather", "Gruber", "Ishikawa"] |
| STREETS = ["Larkspur Lane", "Copperfield Road", "Harbor View Drive", "Old Mill Court", "Juniper Street", "Ridgeway Ave"] |
| CITIES = ["Brookhaven", "Millbrook", "Fairhaven", "Ashford", "Kingsport", "Willow Creek", "Eastlake"] |
| MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", |
| "November", "December"] |
|
|
|
|
| def name(): |
| return f"{rng.choice(FIRST)} {rng.choice(LAST)}" |
|
|
|
|
| def date(): |
| y, m, d = rng.randint(1931, 2026), rng.randint(1, 12), rng.randint(1, 28) |
| return rng.choice([f"{m:02d}/{d:02d}/{y}", f"{MONTHS[m - 1]} {d}, {y}", f"{y}-{m:02d}-{d:02d}", f"{d} {MONTHS[m - 1][:3]} {y}"]) |
|
|
|
|
| def phone(): |
| |
| return rng.choice(["({0}) 555-01{1:02d}", "{0}-555-01{1:02d}", "{0}.555.01{1:02d}", "+1 {0} 555 01{1:02d}"]).format( |
| rng.randint(201, 989), rng.randint(0, 99)) |
|
|
|
|
| def fill(): |
| first, last = rng.choice(FIRST), rng.choice(LAST) |
| return { |
| "NAME": f"{first} {last}", |
| "NAME2": name(), |
| "DATE": date(), |
| "DATE2": date(), |
| "AGE_OVER_89": str(rng.randint(90, 104)), |
| "PHONE": phone(), |
| "FAX": phone(), |
| "EMAIL": rng.choice([f"{first.lower()}.{last.lower()}@example.com", f"{first[0].lower()}{last.lower()}@example.org"]), |
| "SSN": f"9{rng.randint(10, 99)}-{rng.randint(10, 99)}-{rng.randint(1000, 9999)}", |
| "MRN": rng.choice(["MRN ", "MR#", ""]) + str(rng.randint(10000000, 99999999)), |
| "HEALTH_PLAN_ID": rng.choice(["XQH", "BNW", "ZRT"]) + str(rng.randint(100000000, 999999999)), |
| "ACCOUNT_NUMBER": f"ACCT-{rng.randint(100000, 999999)}", |
| "LICENSE_NUMBER": rng.choice(["D", "S", "K"]) + str(rng.randint(1000000, 9999999)), |
| "VEHICLE_ID": rng.choice(["plate ", ""]) + f"{rng.choice('ABCDEFGHJKLMNPRSTUVWXYZ')}{rng.choice('ABCDEFGHJKLMNPRSTUVWXYZ')}{rng.randint(1000, 9999)}", |
| "DEVICE_ID": f"SN-{rng.randint(100000, 999999)}{rng.choice('ABCDEF')}", |
| "URL": rng.choice(["https://portal.example.com/p/", "https://example.org/chart/"]) + str(rng.randint(1000, 9999)), |
| "IP_ADDRESS": f"192.0.2.{rng.randint(1, 254)}", |
| "ADDRESS": f"{rng.randint(10, 9999)} {rng.choice(STREETS)}, {rng.choice(CITIES)}", |
| "ZIP": f"{rng.randint(10000, 99999)}", |
| } |
|
|
|
|
| |
| PHI_TRAIN = [ |
| "Patient {NAME} (DOB {DATE}) called to reschedule her follow-up; best number is {PHONE}.", |
| "Prior authorization for {NAME}, member ID {HEALTH_PLAN_ID}, was denied on {DATE2}.", |
| "Please fax the discharge summary for {MRN} to {FAX} before noon.", |
| "Billing note: account {ACCOUNT_NUMBER} for {NAME} has a balance after the {DATE} visit.", |
| "{NAME} asked that statements go to {ADDRESS} {ZIP} instead of the old address.", |
| "Front desk verified identity with SSN {SSN} and date of birth {DATE}.", |
| "The insulin pump with serial {DEVICE_ID} was replaced for {NAME} under warranty.", |
| "Portal message from {EMAIL}: can I get my lab results before {DATE2}?", |
| "{NAME}, age {AGE_OVER_89}, needs transport; daughter {NAME2} will meet the van.", |
| "Login for the patient portal came from {IP_ADDRESS} at 2 a.m., flagged for review.", |
| "Visit summary link sent to {NAME}: {URL}", |
| "Parking pass issued to {NAME2}, vehicle {VEHICLE_ID}, for the infusion schedule.", |
| "Claim resubmitted for {NAME} with corrected member number {HEALTH_PLAN_ID}.", |
| "Driver's license {LICENSE_NUMBER} copied for the new patient file of {NAME}.", |
| "Referral from Dr. {NAME2} for {NAME}; chart {MRN}; call back at {PHONE}.", |
| "Records request signed by {NAME} on {DATE}; mail copies to {ADDRESS}.", |
| "Superbill for {ACCOUNT_NUMBER} shows two visits, {DATE} and {DATE2}.", |
| "Caregiver {NAME2} can be reached at {EMAIL} or {PHONE}.", |
| "Hearing aid {DEVICE_ID} returned by {NAME} for repair on {DATE2}.", |
| "Eligibility check failed for plan ID {HEALTH_PLAN_ID}; patient {NAME} will bring a new card.", |
| "Telehealth session for {MRN} dropped twice; patient IP {IP_ADDRESS}.", |
| "{NAME} turned {AGE_OVER_89} this year and qualifies for the home visit program.", |
| ] |
| PHI_TEST = [ |
| "Left a voicemail for {NAME} at {PHONE} about the {DATE} appointment.", |
| "Member {HEALTH_PLAN_ID} ({NAME}) disputes the copay on account {ACCOUNT_NUMBER}.", |
| "Chart {MRN} updated: new mailing address {ADDRESS} {ZIP}.", |
| "Pacemaker {DEVICE_ID} check scheduled for {NAME} on {DATE2}; confirm by email at {EMAIL}.", |
| "Spouse {NAME2}, plate {VEHICLE_ID}, is approved for valet at the clinic.", |
| "Identity mismatch: SSN {SSN} on file does not match the one on the claim for {NAME}.", |
| "{NAME} ({AGE_OVER_89}) prefers phone calls; fax records to {FAX}.", |
| "Suspicious portal sign-in from {IP_ADDRESS}; reset link sent: {URL}", |
| ] |
| |
| PHI_NEG_TRAIN = [ |
| "Flu shots are available every Tuesday morning in the main clinic.", |
| "Remind patients that the lab closes at 4 p.m. on holidays.", |
| "The billing team switched to the new claim form in the spring.", |
| "Type 2 diabetes education class moves to room 3 next month.", |
| "Metformin refill requests go to the pharmacy queue, not the front desk.", |
| ] |
| PHI_NEG_TEST = [ |
| "Blood pressure cuffs in exam room 2 need new batteries.", |
| "The prior authorization policy for MRI scans changed this quarter.", |
| ] |
|
|
|
|
| def render(tpl): |
| vals, text, ents, i = fill(), "", [], 0 |
| while i < len(tpl): |
| j = tpl.find("{", i) |
| if j < 0: |
| text += tpl[i:] |
| break |
| k = tpl.index("}", j) |
| text += tpl[i:j] |
| slot = tpl[j + 1:k] |
| value = vals[slot] |
| label = "NAME" if slot == "NAME2" else "DATE" if slot == "DATE2" else slot |
| if slot == "ZIP": |
| label = "ADDRESS" |
| ents.append({"label": label, "start": len(text), "end": len(text) + len(value)}) |
| text += value |
| i = k + 1 |
| return {"text": text, "entities": ents} |
|
|
|
|
| def phi(): |
| train = [render(t) for t in PHI_TRAIN for _ in range(60)] + [render(t) for t in PHI_NEG_TRAIN for _ in range(12)] |
| test = [render(t) for t in PHI_TEST for _ in range(25)] + [render(t) for t in PHI_NEG_TEST for _ in range(10)] |
| rng.shuffle(train) |
| write("hipaa-phi-detector", "train", train) |
| write("hipaa-phi-detector", "test", test) |
|
|
|
|
| |
|
|
| LAW_TRAIN = [ |
| "Our export includes {kw} fields from the last six years.", |
| "Before we license the records, do we need to handle the {kw} data differently?", |
| "The legal team flagged {kw} in the sample set.", |
| "We found {kw} in free-text notes during the scan.", |
| "Which rules apply when the archive holds {kw}?", |
| "A buyer asked how we treat {kw} in the shared files.", |
| "The {kw} records sit in our old system and in email attachments.", |
| "We are cleaning anything tied to {kw} before the review.", |
| ] |
| LAW_TEST = [ |
| "Counsel wants a plan for the {kw} material before any sample goes out.", |
| "Some tickets mention {kw}, so we paused the export.", |
| "How should a company prepare {kw} for an AI data license?", |
| ] |
| |
| LAW_EXTRA = { |
| "HIPAA": ["protected health information", "patient charts", "health plan member IDs", "clinic visit notes"], |
| "42 CFR Part 2": ["addiction treatment records", "substance use disorder notes", "detox program files"], |
| "CCPA and CPRA": ["California consumers' personal information", "a California opt-out of sale request"], |
| "Other US state privacy laws": ["Virginia and Colorado consumer data", "Texas residents' sensitive data"], |
| "GDPR": ["EU customers' personal data", "a data protection officer review in Germany", "consent under EU law"], |
| "UK GDPR": ["UK customers' personal data", "an ICO complaint", "data from our London office"], |
| "PIPEDA and Quebec Law 25": ["Canadian customers' personal information", "Quebec privacy officer approval"], |
| "GLBA and the FTC Safeguards Rule": ["bank customer account data", "loan applicant financial details"], |
| "FCRA": ["tenant screening reports", "employment background check results", "credit report pulls"], |
| "COPPA": ["accounts of kids under 13", "parent consent forms for a children's app"], |
| "PCI DSS": ["full credit card numbers", "cardholder data and CVV codes", "card expiry dates"], |
| "CAN-SPAM": ["marketing email unsubscribe lists", "newsletter opt-out records"], |
| "Fair Housing Act": ["rental applicant familial status", "tenant disability accommodation requests"], |
| "Driver's Privacy Protection Act": ["DMV records", "license plate lookups", "driver license scans"], |
| "ITAR and EAR": ["export-controlled technical data", "defense article drawings", "ECCN-classified designs"], |
| "CUI and DFARS": ["controlled unclassified information from a defense contract", "NIST 800-171 scoped files"], |
| "GINA": ["genetic test results", "employee family medical history", "hereditary condition screening"], |
| "21 CFR Part 11": ["electronic batch records with signatures", "GxP audit trails", "validated lab system logs"], |
| "Attorney-client privilege and professional conduct rules": ["privileged client emails", "attorney work product"], |
| "Contractual confidentiality and NDAs": ["files covered by a customer NDA", "MSA-restricted client deliverables"], |
| "Trade secret protection": ["our proprietary process recipes", "confidential product designs", "manufacturing know-how"], |
| } |
|
|
|
|
| def laws(): |
| fws = read(DS / "data-compliance-frameworks/data/frameworks.jsonl") |
| train, test = [], [] |
| for fw in fws: |
| label = fw["framework"] |
| cues = list(fw["keywords"]) + LAW_EXTRA[label] |
| for kw in cues: |
| train += [{"text": t.format(kw=kw), "label": label} for t in LAW_TRAIN] |
| test += [{"text": t.format(kw=kw), "label": label} for t in LAW_TEST] |
| rng.shuffle(train) |
| write("privacy-law-classifier", "train", train) |
| write("privacy-law-classifier", "test", test) |
|
|
|
|
| |
|
|
| def record_types(): |
| pairs = read(DS / "synthetic-deidentification-pairs/data/train.jsonl") |
| rows = [{"text": p["text"], "label": p["kind"]} for p in pairs] |
| rng.shuffle(rows) |
| write("business-record-type-classifier", "train", rows[:480]) |
| write("business-record-type-classifier", "test", rows[480:]) |
|
|
|
|
| |
|
|
| def tickets(): |
| rows = read(DS / "synthetic-support-tickets-with-outcomes/data/train.jsonl") |
| out = [] |
| for r in rows: |
| convo = "\n".join(f"{m['role']}: {m['text']}" for m in r["messages"]) |
| out.append({"text": convo, "label": r["resolution"]["outcome"], "category": r["category"]}) |
| rng.shuffle(out) |
| write("support-ticket-outcome-classifier", "train", out[:400]) |
| write("support-ticket-outcome-classifier", "test", out[400:]) |
|
|
|
|
| if __name__ == "__main__": |
| pii() |
| phi() |
| laws() |
| record_types() |
| tickets() |
|
|
