Add source/gen_synth.py

source/gen_synth.py

@@ -0,0 +1,379 @@
+"""Datasets 5 to 10 and the training files of models 4 to 10: invented text, no real person, company or secret (2026-10-30).
+
+    python hf-work/gen_synth.py
+Each dataset is filled from templates. The TEST templates are never used for training, so a score measures new sentence
+shapes and not only new names. Names, companies and numbers are invented; phone numbers use the fictional 555-01xx range,
+emails use example domains, and keys carry a "demo" prefix.
+"""
+
+import json
+import random
+import string
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+rng = random.Random(20261030)
+
+FIRST = ["Marisol", "Dennick", "Aurelio", "Tamsin", "Liesel", "Bram", "Yusra", "Corwin", "Ilse", "Nadim", "Petra", "Ravindra",
+         "Sunniva", "Teodor", "Wren", "Zaida", "Hollis", "Imogen", "Kasimir", "Delphine", "Ezio", "Odalys", "Fenwick", "Galen"]
+LAST = ["Quarrington", "Abernethy", "Okafor", "Lindqvist", "Marchetti", "Pemberton", "Szabo", "Thistlewood", "Hakimi",
+        "Brannigan", "Castellanos", "Duchesne", "Fairweather", "Gruber", "Ishikawa", "Valdespino", "Nwosu", "Ostrander"]
+ORGS = ["Larkspur Logistics", "Copperfield Supply", "Harborview Dental", "Northgate Analytics", "Juniper Freight", "Ridgeway Builders",
+        "Millbrook Software", "Ashford Insurance Group", "Kingsport Marine", "Willow Creek Foods", "Eastlake Clinics",
+        "Brightwater Staffing", "Granite Peak Capital", "Fernhill Media", "Stonebridge Legal", "Tidewater Robotics"]
+STREETS = ["Larkspur Lane", "Copperfield Road", "Harbor View Drive", "Old Mill Court", "Juniper Street", "Ridgeway Avenue",
+           "Elmhurst Boulevard", "Quarry Road", "Saltmarsh Way", "Pine Terrace"]
+CITIES = ["Brookhaven", "Millbrook", "Fairhaven", "Ashford", "Kingsport", "Willow Creek", "Eastlake", "Stonebridge", "Tidewater"]
+MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
+DOMAINS = ["example.com", "example.org", "mail.example", "corp.example", "sample.test"]
+
+
+def person():
+    return f"{rng.choice(FIRST)} {rng.choice(LAST)}"
+
+
+def org():
+    return rng.choice(ORGS)
+
+
+def email(name=None):
+    n = (name or person()).lower().replace(" ", ".")
+    return f"{n}@{rng.choice(DOMAINS)}"
+
+
+def phone():
+    return f"(555) 01{rng.randint(0, 9)}-{rng.randint(1000, 9999)}"
+
+
+def address():
+    return f"{rng.randint(12, 980)} {rng.choice(STREETS)}, {rng.choice(CITIES)}"
+
+
+def date():
+    return f"{rng.choice(MONTHS)} {rng.randint(1, 28)}, {rng.randint(2024, 2026)}"
+
+
+def money():
+    return f"${rng.randint(120, 48000):,}.{rng.randint(0, 99):02d}"
+
+
+def code(prefix, n=6):
+    return prefix + "".join(rng.choice(string.digits) for _ in range(n))
+
+
+def key():
+    return "demo_key_" + "".join(rng.choice(string.ascii_lowercase + string.digits) for _ in range(20))
+
+
+def ip():
+    return f"10.{rng.randint(0, 250)}.{rng.randint(0, 250)}.{rng.randint(2, 250)}"
+
+
+def write(kind, name, split, rows):
+    out = ROOT / kind / name / "data"
+    out.mkdir(parents=True, exist_ok=True)
+    with open(out / f"{split}.jsonl", "w", encoding="utf-8", newline="\n") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    print(f"{kind}/{name}/{split}: {len(rows)} rows")
+
+
+PRE = ["", "", "Note: ", "FYI, ", "Quick one: ", "Following up. ", "Per our earlier message, ", "Hello. ", "To whom it may concern: ", "Sorry for the delay. "]
+SUF = ["", "", " Thanks.", " Please advise.", " Let me know.", " More detail on request.", " Regards.", " Appreciated."]
+
+
+def classification(name, label_field, train_t, test_t, fill, per_template, extra=None):
+    """train_t / test_t: {label: [template, ...]}. Returns (train_rows, test_rows) and writes the dataset."""
+    out = {}
+    for split, templates in (("train", train_t), ("test", test_t)):
+        rows, seen = [], set()
+        for label, tpls in templates.items():
+            for t in tpls:
+                for _ in range(per_template):
+                    text = rng.choice(PRE) + t.format(**fill()) + rng.choice(SUF)
+                    if text in seen:
+                        continue
+                    seen.add(text)
+                    rows.append({"text": text, label_field: label, **({k: v(label) for k, v in extra.items()} if extra else {})})
+        rng.shuffle(rows)
+        out[split] = rows
+    return out
+
+
+def with_ids(prefix, rows):
+    return [{"id": f"{prefix}-{i:04d}", **r} for i, r in enumerate(rows, 1)]
+
+
+def spans_from(segments):
+    text, ents = "", []
+    for seg, lab in segments:
+        if lab:
+            ents.append({"label": lab, "start": len(text), "end": len(text) + len(seg)})
+        text += seg
+    return {"text": text, "entities": ents}
+
+
+# ======================= dataset 5 and model 9: document sensitivity tier =======================
+THING = ["the Q3 roadmap", "the onboarding guide", "the vendor list", "the pricing sheet", "the incident review", "the hiring plan",
+         "the support playbook", "the launch plan", "the audit notes", "the budget draft", "the partner terms", "the training deck"]
+TEAM = ["support", "finance", "engineering", "sales", "operations", "legal", "people team", "marketing"]
+TIER_FILL = lambda: {"thing": rng.choice(THING), "team": rng.choice(TEAM), "org": org(), "n": rng.randint(3, 40), "month": rng.choice(MONTHS)}
+TIER_TRAIN = {
+    "public": ["{org} announces {thing} in a press release for all media.", "Blog post for our website readers: how {team} works with customers.",
+               "Published job listing for a {team} role, visible on our careers page.", "Public FAQ answer about opening hours and how to contact {team}.",
+               "Product brochure PDF for download, approved by marketing for anyone.", "Social media post celebrating {n} years in business.",
+               "Open-source release notes for version {n}, posted on our public repository.", "Conference talk slides cleared for the public audience."],
+    "internal": ["Team update from {team}: {thing} moves to next sprint, share inside the company only.", "All-hands notes from {month}, for employees, not for customers.",
+                 "Internal wiki page on how {team} files expense reports.", "Slack thread where {team} compares tools; staff only.",
+                 "Meeting agenda for the weekly {team} sync and action items.", "Draft of {thing} circulating among managers for comment.",
+                 "Staff newsletter with office closures and training dates.", "Org chart and desk map for employees."],
+    "confidential": ["Customer contract with {org} that names pricing, renewal terms and discounts.", "Revenue forecast for the board, not for release outside leadership.",
+                     "Unreleased {thing}, under embargo until the announcement.", "Supplier agreement with negotiated rates for {team}.",
+                     "Investor update with cash position and runway for {n} months.", "Source code of the billing service, access limited to engineers.",
+                     "Acquisition discussion notes with {org}, known to a handful of executives.", "Salary bands and bonus plan for {team}, shared with HR only."],
+    "restricted": ["Patient records with names, diagnoses and insurance numbers, covered by health privacy law.", "Export of customer payment card numbers and billing addresses.",
+                   "Employee Social Security numbers and bank details held by payroll.", "Production database credentials and private signing keys for {thing}.",
+                   "Legal hold material under attorney-client privilege for a pending case.", "Unredacted identity documents scanned for {n} customers.",
+                   "Full customer database with emails, phone numbers and home addresses, un-de-identified.", "Security incident forensics naming affected accounts and exploited weaknesses."],
+}
+TIER_TEST = {
+    "public": ["Newsletter issue for subscribers about {thing}, published openly.", "Open house invitation from {org}, posted for the neighbourhood."],
+    "internal": ["Reminder to {team} staff about the holiday schedule and who covers the desk.", "Notes from {team}'s retrospective on {thing}, inside the company."],
+    "confidential": ["Term sheet from {org}, only for the deal team.", "Customer churn analysis with account names and contract values."],
+    "restricted": ["Spreadsheet of staff medical leave reasons and doctors' notes.", "Raw customer list with government ID numbers attached to {n} rows."],
+}
+r = classification("document-sensitivity-tiers", "label", TIER_TRAIN, TIER_TEST, TIER_FILL, 10)
+write("datasets", "synthetic-document-sensitivity-tiers", "train", with_ids("tier", [{"text": x["text"], "tier": x["label"]} for x in r["train"]]))
+write("datasets", "synthetic-document-sensitivity-tiers", "test", with_ids("tier-t", [{"text": x["text"], "tier": x["label"]} for x in r["test"]]))
+write("models", "document-sensitivity-tier-classifier", "train", r["train"])
+write("models", "document-sensitivity-tier-classifier", "test", r["test"])
+
+# ======================= dataset 6, models 4 and 10: contract clauses (type and risk) =======================
+CL_FILL = lambda: {"a": rng.choice(["the Seller", "the Company", "Licensor"]), "b": rng.choice(["the Buyer", "the Lab", "Licensee"]),
+                   "n": rng.choice([30, 45, 60, 90, 180]), "y": rng.choice([1, 2, 3, 5]), "p": rng.choice(["thirty", "forty-five", "sixty", "ninety"])}
+CLAUSES = {  # type -> (standard train, watch train, standard test, watch test)
+    "exclusivity": (["{b} receives exclusive rights to the Dataset for {n} days, after which {a} may license it to others.",
+                     "During the exclusivity period of {n} days, {a} will not license the Dataset to any other party; the period ends on a fixed date."],
+                    ["{b} receives exclusive rights to the Dataset for all purposes, with no end date.",
+                     "{a} may never license the Dataset or anything derived from it to any other party, in any field."],
+                    ["Exclusivity applies for {y} year(s) and only to the named use, and {a} keeps all other rights."],
+                    ["The exclusive grant covers every future version of the Dataset and renews automatically without notice."]),
+    "term_and_renewal": (["This Agreement lasts {y} year(s) and renews only if both parties agree in writing.",
+                          "The term is {y} year(s) from the effective date, and either party may decline renewal with {n} days' notice."],
+                         ["This Agreement renews automatically for successive {y}-year terms and {a} cannot end it.",
+                          "The term continues until {b} decides to stop, at its sole discretion."],
+                         ["The initial term is {y} year(s); renewal needs a signed amendment."],
+                         ["The license is perpetual and irrevocable and survives any breach by {b}."]),
+    "payment": (["{b} pays the fee within {n} days of an invoice, by bank transfer, and late amounts accrue modest interest.",
+                 "The fee is due in stages: a deposit on signing and the balance on acceptance of the delivered data."],
+                ["{b} may withhold payment for any reason until it is fully satisfied, with no deadline.",
+                 "Payment is due only after {b} has earned revenue from a model trained on the Dataset, and {a} cannot see those figures."],
+                ["Invoices are payable within {n} days and disputes must be raised in writing within ten days."],
+                ["Fees may be reduced by {b} at any time by notice, and refunds are due on request after delivery."]),
+    "confidentiality": (["Each party keeps the other's confidential information private for {y} year(s) after the Agreement ends, with the usual exceptions.",
+                         "{b} will protect the Dataset with the care it gives its own secrets and limit access to those who need it."],
+                        ["{a} must keep every detail of this Agreement secret forever, even from its own advisers and auditors.",
+                         "{b} may disclose the Dataset to any third party without telling {a}."],
+                        ["Confidential information excludes what is public, already known or independently developed."],
+                        ["Confidentiality binds {a} only; {b} owes no duty of secrecy about anything it learns."]),
+    "ip_and_ownership": (["{a} keeps ownership of the Dataset and grants {b} a limited license for the stated purpose.",
+                          "Models {b} trains remain {b}'s, and {a} keeps its rights in the source records."],
+                         ["Ownership of the Dataset passes to {b} on delivery, including all future records {a} creates.",
+                          "{b} owns everything {a} produces during the term, whether or not it relates to the Dataset."],
+                         ["The license is non-transferable and covers only the field of use set out in the schedule."],
+                         ["{a} assigns all present and future rights in its systems and records to {b} for no extra fee."]),
+    "indemnity": (["Each party defends the other against claims caused by its own breach, up to a stated cap.",
+                   "{a} stands behind its statement that it has the right to license the Dataset, with liability capped at the fees paid."],
+                  ["{a} indemnifies {b} without any cap for every claim of any kind, including claims {b} causes.",
+                   "{a} pays {b}'s legal costs on any dispute, even where {a} wins."],
+                  ["Liability under this clause is limited to the fees paid in the {n} days before the claim."],
+                  ["{a} bears all liability for how {b} uses the Dataset after delivery, with no limit."]),
+    "audit_and_deletion": (["{a} may ask once a year for a written confirmation of how the Dataset is used, and {b} deletes it at the end of the term.",
+                            "On request {b} confirms in writing that copies were deleted within {n} days."],
+                           ["{b} may keep the Dataset forever and {a} has no way to check how it is used.",
+                            "{b} may share the Dataset with its affiliates and partners and need not delete it."],
+                           ["At the end of the term {b} deletes the Dataset and certifies it, and {a} may audit once with notice."],
+                           ["There is no audit right and no duty to delete, even if {a} ends the Agreement for breach."]),
+    "termination": (["Either party may end the Agreement for uncured breach after {n} days' written notice.",
+                     "{a} may end the Agreement if {b} misses a payment and does not fix it within {n} days."],
+                    ["Only {b} may end the Agreement, at any time, and keeps everything delivered.",
+                     "{a} cannot end the Agreement for any reason, including non-payment."],
+                    ["Termination for convenience needs {n} days' notice, and fees already earned stay payable."],
+                    ["On termination by {b}, all data already delivered stays with {b} and no fees are refunded or owed."]),
+}
+tr_t_type, te_t_type, tr_t_risk, te_t_risk = {}, {}, {"standard": [], "watch": []}, {"standard": [], "watch": []}
+rows_train, rows_test = [], []
+for ctype, (s_tr, w_tr, s_te, w_te) in CLAUSES.items():
+    for tpls, risk, rows in ((s_tr, "standard", rows_train), (w_tr, "watch", rows_train), (s_te, "standard", rows_test), (w_te, "watch", rows_test)):
+        seen = set()
+        for t in tpls:
+            for _ in range(12):
+                text = rng.choice(PRE) + t.format(**CL_FILL()) + rng.choice(SUF)
+                if text not in seen:
+                    seen.add(text)
+                    rows.append({"text": text, "clause_type": ctype, "risk": risk})
+rng.shuffle(rows_train)
+rng.shuffle(rows_test)
+write("datasets", "synthetic-contract-clauses-by-type-and-risk", "train", with_ids("cl", rows_train))
+write("datasets", "synthetic-contract-clauses-by-type-and-risk", "test", with_ids("cl-t", rows_test))
+write("models", "contract-clause-type-classifier", "train", [{"text": x["text"], "label": x["clause_type"]} for x in rows_train])
+write("models", "contract-clause-type-classifier", "test", [{"text": x["text"], "label": x["clause_type"]} for x in rows_test])
+write("models", "contract-clause-risk-classifier", "train", [{"text": x["text"], "label": x["risk"]} for x in rows_train])
+write("models", "contract-clause-risk-classifier", "test", [{"text": x["text"], "label": x["risk"]} for x in rows_test])
+
+# ======================= dataset 7 and model 5: inbound inquiries by intent =======================
+IQ_FILL = lambda: {"name": person(), "org": org(), "n": rng.choice([30, 45, 60, 120, 400]), "thing": rng.choice(
+    ["support tickets", "CRM history", "design files", "call recordings", "team chat", "project records", "code reviews", "invoices"])}
+INTENT_TRAIN = {
+    "seller_interest": ["Hi, I run {org} with about {n} staff. We have years of {thing}. Could we sell or license it to an AI lab?",
+                        "My company has a lot of {thing} sitting unused. How does licensing it work and what would you need from us?",
+                        "We are closing down {org} and want to know whether our {thing} has any value to AI companies.",
+                        "I saw your video. Does a firm of {n} people qualify, and who pays your fee?",
+                        "Our owners are open to licensing {thing}. What is the first step?", "Interested in learning how to monetise our archive of {thing}. Please call me."],
+    "buyer_request": ["We are an AI lab looking for licensed {thing} from operating companies. Can you share what you have?",
+                      "Our model team needs a dataset of {thing}, de-identified, with clear rights. Do you source this?",
+                      "Looking to buy business records for training an agent. Which industries can you cover?",
+                      "As a buyer we want sample data before committing. How do you handle diligence?",
+                      "Please send what you can offer in {thing}; we have a budget this quarter.", "Our research group wants to license real-world {thing} and needs the rights confirmed."],
+    "referral_offer": ["I know several owners with {thing}. How does your referral arrangement work?",
+                       "As a consultant I can introduce companies of {n} people. What do referrers receive?",
+                       "My accounting clients keep asking about selling their data. Can we partner on introductions?",
+                       "I run an M&A advisory and can bring wind-down companies. Do you work with referral partners?",
+                       "Happy to introduce owners if the terms are fair. Please send your partner details.", "We have a list of companies that may want to license {thing}; can we share it with you?"],
+    "press_media": ["I am a reporter writing about AI training data. Could you give a comment on how licensing works?",
+                    "Our podcast would like to interview you about selling company data to AI labs.",
+                    "Writing a feature on data brokers for our newsletter; can we schedule a short call?",
+                    "Journalist here. Do you have figures on how many companies license their data?",
+                    "We would like to quote you in an article about {thing} and AI.", "An editor asked me to request an interview about your work."],
+    "vendor_pitch": ["We offer lead generation and can fill your calendar with {n} meetings a month.", "Our agency does SEO and link building for sites like yours. Want a free audit?",
+                     "I can build you a website redesign at a discount this week.", "Our software helps companies like {org} automate outreach. Book a demo?",
+                     "We sell guest posts on high authority blogs; interested?", "Special offer on email lists for your campaigns, limited time."],
+    "support_question": ["I filled in your form but did not get a reply. Can you check on it?", "How do I update the details I sent you last week?",
+                         "I cannot open the calculator result page. Could you help?", "Where do I find the checklist you mentioned in the video?",
+                         "I made an account, but I forgot which email I used. Can you help me sign in?", "The page says error when I submit my company information."],
+    "opt_out": ["Please remove me from your list.", "Stop emailing me. I am not interested.", "Unsubscribe me and do not contact this address again.",
+                "I did not ask for these messages. Take me off your list.", "Not interested, please delete my details.", "Remove {org} from your mailing list."],
+    "spam": ["Congratulations, you have won a prize, click here to claim.", "Buy cheap watches and pills now, best prices guaranteed.",
+             "Earn money fast from home, no skills needed, send your bank login.", "Your account is suspended, verify your password at the link.",
+             "Hot singles near you want to meet tonight.", "Crypto giveaway, send one coin and get ten back."],
+}
+INTENT_TEST = {
+    "seller_interest": ["We own a small agency, {n} employees, with {thing} going back years. Is there a market for this?", "Can {org} get paid for its {thing}? We would like an honest answer."],
+    "buyer_request": ["Procurement at our lab is exploring licensed {thing}; what is your sourcing process?", "We are building an evaluation set and need real {thing} with permission."],
+    "referral_offer": ["Do you pay a finder's fee if I connect you with a company of {n} staff?", "I advise owners on exits and would like to discuss working together on introductions."],
+    "press_media": ["I cover AI for a trade magazine and need an expert on data licensing for a quote.", "Would you join our webinar to explain how companies sell {thing} to AI labs?"],
+    "vendor_pitch": ["We help companies like {org} get more traffic with our paid promotion package.", "Our team can create videos for your channel at a fraction of the usual cost."],
+    "support_question": ["The link in your email does not work for me. Can you resend it?", "I need to correct a number I entered on the form. How do I do that?"],
+    "opt_out": ["Kindly take {org} off all further messages.", "I would like no further contact. Please confirm you removed me."],
+    "spam": ["Winner! Claim your free gift card by entering your card details here.", "Invest now and double your money in a week, guaranteed returns."],
+}
+r = classification("inquiries", "label", INTENT_TRAIN, INTENT_TEST, IQ_FILL, 10)
+write("datasets", "synthetic-inbound-inquiries-by-intent", "train", with_ids("inq", [{"text": x["text"], "intent": x["label"]} for x in r["train"]]))
+write("datasets", "synthetic-inbound-inquiries-by-intent", "test", with_ids("inq-t", [{"text": x["text"], "intent": x["label"]} for x in r["test"]]))
+write("models", "inbound-inquiry-intent-classifier", "train", r["train"])
+write("models", "inbound-inquiry-intent-classifier", "test", r["test"])
+
+# ======================= dataset 8 and model 6: invoice and purchase order text with personal and account details =======================
+def inv_fill():
+    return {"p": person(), "o": org(), "e": email(), "ph": phone(), "ad": address(), "d": date(), "m": money(), "i": code("INV-"), "ac": code("", 10), "t": code("", 9)}
+
+
+def spans_template(parts):
+    """parts: list of (text-or-slot, label). Slot names in braces fill from `fill`."""
+    return parts
+
+
+INV_TRAIN = [
+    [("Invoice ", None), ("{i}", "INVOICE_ID"), (" issued ", None), ("{d}", "DATE"), (" to ", None), ("{o}", "ORG"), (", attn ", None), ("{p}", "PERSON"), (". Amount due ", None), ("{m}", "MONEY"), (".", None)],
+    [("Bill to: ", None), ("{p}", "PERSON"), (", ", None), ("{ad}", "ADDRESS"), (". Contact ", None), ("{e}", "EMAIL"), (" or ", None), ("{ph}", "PHONE"), (". Total ", None), ("{m}", "MONEY"), (".", None)],
+    [("Please remit ", None), ("{m}", "MONEY"), (" to account ", None), ("{ac}", "BANK_ACCOUNT"), (" by ", None), ("{d}", "DATE"), (" quoting ", None), ("{i}", "INVOICE_ID"), (".", None)],
+    [("Vendor: ", None), ("{o}", "ORG"), (" (tax ID ", None), ("{t}", "TAX_ID"), ("). Remit to ", None), ("{ad}", "ADDRESS"), (".", None)],
+    [("Purchase order ", None), ("{i}", "INVOICE_ID"), (" approved by ", None), ("{p}", "PERSON"), (" on ", None), ("{d}", "DATE"), (" for ", None), ("{m}", "MONEY"), (".", None)],
+    [("Questions about this invoice? Email ", None), ("{e}", "EMAIL"), (" or call ", None), ("{ph}", "PHONE"), (". Ref ", None), ("{i}", "INVOICE_ID"), (".", None)],
+    [("Payment received from ", None), ("{o}", "ORG"), (" on ", None), ("{d}", "DATE"), (", ", None), ("{m}", "MONEY"), (", from account ending ", None), ("{ac}", "BANK_ACCOUNT"), (".", None)],
+]
+INV_TEST = [
+    [("Statement for ", None), ("{p}", "PERSON"), (" at ", None), ("{o}", "ORG"), (": balance ", None), ("{m}", "MONEY"), (", due ", None), ("{d}", "DATE"), (".", None)],
+    [("Wire details: account ", None), ("{ac}", "BANK_ACCOUNT"), (", beneficiary ", None), ("{o}", "ORG"), (", tax number ", None), ("{t}", "TAX_ID"), (".", None)],
+    [("Ship to ", None), ("{ad}", "ADDRESS"), (" for ", None), ("{p}", "PERSON"), (", phone ", None), ("{ph}", "PHONE"), (", order ", None), ("{i}", "INVOICE_ID"), (".", None)],
+    [("Credit note ", None), ("{i}", "INVOICE_ID"), (" of ", None), ("{m}", "MONEY"), (" sent to ", None), ("{e}", "EMAIL"), (" on ", None), ("{d}", "DATE"), (".", None)],
+]
+
+
+def render_spans(templates, per, fill):
+    rows, seen = [], set()
+    for tpl in templates:
+        for _ in range(per):
+            f = fill()
+            seg = []
+            for text, lab in tpl:
+                seg.append((text.format(**f) if "{" in text else text, lab))
+            row = spans_from(seg)
+            if row["text"] not in seen:
+                seen.add(row["text"])
+                rows.append(row)
+    rng.shuffle(rows)
+    return rows
+
+
+tr, te = render_spans(INV_TRAIN, 40, inv_fill), render_spans(INV_TEST, 30, inv_fill)
+write("datasets", "synthetic-invoice-pii-spans", "train", with_ids("inv", tr))
+write("datasets", "synthetic-invoice-pii-spans", "test", with_ids("inv-t", te))
+write("models", "invoice-pii-detector", "train", tr)
+write("models", "invoice-pii-detector", "test", te)
+
+# ======================= dataset 9 and model 7: chat logs with personal and technical identifiers =======================
+def chat_fill():
+    return {"p": person(), "p2": person(), "e": email(), "ph": phone(), "k": key(), "ip": ip(), "t": code("TCK-", 5)}
+
+
+CHAT_TRAIN = [
+    [("[10:42] ", None), ("{p}", "PERSON"), (": can you email the file to ", None), ("{e}", "EMAIL"), (" before lunch?", None)],
+    [("[14:03] ", None), ("{p}", "PERSON"), (": customer rang from ", None), ("{ph}", "PHONE"), (", ticket is ", None), ("{t}", "ID_NUMBER"), (".", None)],
+    [("oops I pasted the key in here: ", None), ("{k}", "API_KEY"), (" please rotate it", None)],
+    [("{p}", "PERSON"), (" > the server at ", None), ("{ip}", "IP_ADDRESS"), (" keeps timing out again", None)],
+    [("@", None), ("{p2}", "PERSON"), (" can you take ", None), ("{t}", "ID_NUMBER"), ("? ", None), ("{p}", "PERSON"), (" is out today", None)],
+    [("Reach me on ", None), ("{ph}", "PHONE"), (" if the deploy fails, or ", None), ("{e}", "EMAIL"), (".", None)],
+    [("config line: token=", None), ("{k}", "API_KEY"), (" host=", None), ("{ip}", "IP_ADDRESS")],
+    [("thanks ", None), ("{p2}", "PERSON"), (", closing ", None), ("{t}", "ID_NUMBER"), (" now", None)],
+]
+CHAT_TEST = [
+    [("DM from ", None), ("{p}", "PERSON"), (": my login is ", None), ("{e}", "EMAIL"), (", the old box is ", None), ("{ip}", "IP_ADDRESS")],
+    [("Heads up ", None), ("{p2}", "PERSON"), (": api secret ", None), ("{k}", "API_KEY"), (" is in the wiki page, remove it", None)],
+    [("call ", None), ("{p}", "PERSON"), (" on ", None), ("{ph}", "PHONE"), (" about ", None), ("{t}", "ID_NUMBER")],
+    [("{p}", "PERSON"), (" and ", None), ("{p2}", "PERSON"), (" are both cc'd at ", None), ("{e}", "EMAIL")],
+]
+tr, te = render_spans(CHAT_TRAIN, 45, chat_fill), render_spans(CHAT_TEST, 35, chat_fill)
+write("datasets", "synthetic-chat-logs-pii-spans", "train", with_ids("chat", tr))
+write("datasets", "synthetic-chat-logs-pii-spans", "test", with_ids("chat-t", te))
+write("models", "chat-log-pii-detector", "train", tr)
+write("models", "chat-log-pii-detector", "test", te)
+
+# ======================= dataset 10 and model 8: sales call notes by outcome =======================
+CALL_FILL = lambda: {"name": person(), "org": org(), "n": rng.choice([2, 3, 4, 6]), "w": rng.choice(["Friday", "next Tuesday", "the 14th", "end of month"]),
+                     "thing": rng.choice(["the pilot", "the proposal", "the quote", "the demo", "the sample"])}
+OUT_TRAIN = {
+    "advanced": ["Spoke with {name} at {org}. Liked {thing}, wants a second call with the team on {w}.", "{name} asked for a written proposal and named two colleagues to join the next meeting.",
+                 "Good fit confirmed. {org} will send access for {thing} and we booked a review on {w}.", "Budget approved in principle by {name}; next step is the security questionnaire.",
+                 "{name} introduced us to legal at {org}. Drafting terms this week.", "Strong interest. They want {thing} reworked for their data and will reply by {w}."],
+    "stalled": ["{name} said the team is busy and to check back after {w}. No date set.", "Waiting on {org} to reply to the last {n} emails. Nothing since the call.",
+                "{name} likes it but needs sign-off from a person they cannot name yet.", "Pushed back to next quarter by {org}; no owner for {thing}.",
+                "Left a voicemail twice for {name}. Still no answer on {thing}.", "Interest is real but their priorities changed and the project is on hold."],
+    "lost": ["{name} told us {org} chose another provider and will not continue.", "Price was too high for {org}; {name} closed the file.",
+             "{org} decided to build it in house, so no further calls are needed.", "{name} said it is not a fit for their size and asked us to stop following up.",
+             "Legal at {org} refused the data terms and the deal ended.", "{name} replied that they have no interest in {thing} and no budget now or later."],
+    "won": ["{name} signed the agreement for {org} and we start on {w}.", "Contract returned countersigned by {org}; invoice sent for {thing}.",
+            "{org} confirmed the order after {thing} and payment is scheduled for {w}.", "{name} approved the final terms; kickoff booked with {n} people from their side.",
+            "Deal closed. {org} signed, and first delivery is planned for {w}.", "Signed today. {name} thanked the team and asked for the onboarding pack."],
+}
+OUT_TEST = {
+    "advanced": ["{name} wants {thing} sent to their technical lead before the next call with {org}.", "They asked for references and set a follow-up for {w}."],
+    "stalled": ["No response from {org} after the proposal; last contact was {n} weeks ago.", "{name} is travelling and said to try again later without giving a date."],
+    "lost": ["{org} went with a cheaper alternative, according to {name}.", "{name} said the project was cancelled and {org} no longer needs this."],
+    "won": ["{org} has accepted the quote and {name} sent the purchase order.", "We got the yes from {name}; paperwork is complete."],
+}
+r = classification("calls", "label", OUT_TRAIN, OUT_TEST, CALL_FILL, 10)
+write("datasets", "synthetic-sales-call-notes-by-outcome", "train", with_ids("call", [{"text": x["text"], "outcome": x["label"]} for x in r["train"]]))
+write("datasets", "synthetic-sales-call-notes-by-outcome", "test", with_ids("call-t", [{"text": x["text"], "outcome": x["label"]} for x in r["test"]]))
+write("models", "sales-call-outcome-classifier", "train", r["train"])
+write("models", "sales-call-outcome-classifier", "test", r["test"])