| """Build every dataset Sanad needs, in one go. |
| |
| python scripts/make_all_data.py |
| |
| data/ |
| operator/ operator_train.jsonl operator_val.jsonl operator_eval.jsonl (120 held-out scenarios) |
| guardian/ guardian_train.jsonl guardian_val.jsonl guardian_eval.jsonl (252 review packets + expected decision) |
| verifier/ verifier_train.jsonl verifier_val.jsonl verifier_eval.jsonl (re-checks + expected status) |
| stats.json counts, label balance, diversity and length |
| Each agent is trained separately (its own LoRA adapter on the same base model). |
| |
| Every row: {"messages": [...], "tools": [...], "meta": {"role": ..., ...}} |
| Names of hosts, VMs, datastores, cluster and vCenter change in every episode. |
| """ |
| from __future__ import annotations |
|
|
| import json |
| import random |
| import sys |
| from collections import Counter |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| sys.path.insert(0, str(ROOT)) |
|
|
| from sanad_sim import FAULTS, Simulator |
| from sanad_sim.checks import check_grounding |
| from sanad_sim.guardian_data import EVAL_MODES, MODE_CYCLE, GuardianBuilder |
| from sanad_sim.teacher import ScriptedTeacher |
| from sanad_sim.verifier_data import CASE_CYCLE, CASES as V_EVAL_CASES, VerifierBuilder |
|
|
| FIDS = list(FAULTS) |
| SEEDS = { |
| "operator": {"train": 0, "val": 500_000, "eval": 900_000}, |
| "guardian": {"train": 1_100_000, "val": 1_500_000, "eval": 1_900_000}, |
| "verifier": {"train": 2_100_000, "val": 2_500_000, "eval": 2_900_000}, |
| } |
| SIZES = {"operator": {"train": 70, "val": 8}, "guardian": {"train": 80, "val": 10}, |
| "verifier": {"train": 40, "val": 5}} |
| SPECIAL_SIZES = {"train": {"OK00": 150, "UNK00": 100}, "val": {"OK00": 20, "UNK00": 15}} |
| P_TOOL_ERROR = 0.15 |
|
|
|
|
| |
| def operator_row(fid, seed, lang, teacher): |
| rng = random.Random(seed * 7919 + 13) |
| probe = Simulator(seed).new_episode(fid, lang=lang) |
| errs = None |
| if fid not in ("UNK00",) and rng.random() < P_TOOL_ERROR: |
| errs = [{"tool": probe.inj.evidence[0][0], "call": 1, |
| "error": rng.choice(["timeout after 30s", "connection reset by vCenter", "503 Service Unavailable"])}] |
| ep = Simulator(seed).new_episode(fid, lang=lang, tool_errors=errs) |
| s = teacher.run(ep, rng) |
| fa = s["first_action"] |
| if fid == "UNK00": |
| if check_grounding(s["final_answer"], ep.trace): |
| return None, "grounding" |
| else: |
| chk = ep.code_checks(s["final_answer"], fa and fa["action"], fa and fa["args"]) |
| if not chk["pass"]: |
| return None, f"checks {[i['type'] for i in chk['issues']]}" |
| for action, args in ep.inj.solution: |
| ep.execute(action, args) |
| if not ep.resolved(): |
| return None, "fix did not resolve" |
| msgs = ep.display(s["messages"]) |
| |
| shown = [{"output": json.loads(m["content"])} for m in msgs if m["role"] == "tool"] |
| shown += [{"tool": "propose_action", "args": tc["function"]["arguments"], "output": {}} |
| for m in msgs if m["role"] == "assistant" for tc in m.get("tool_calls", []) |
| if tc["function"]["name"] == "propose_action"] |
| if fid != "UNK00" and check_grounding(msgs[-1]["content"], shown): |
| return None, "grounding after rename" |
| meta = {"role": "operator", "fault_id": fid, "seed": seed, "lang": lang, "tool_error": bool(errs), |
| "ground_truth": ep.display(ep.ground_truth())} |
| return {"messages": msgs, "tools": s["tools"], "meta": meta}, None |
|
|
|
|
| def build_operator(split, teacher, drops): |
| rows, base, n = [], SEEDS["operator"][split], SIZES["operator"][split] |
| plan = [(fid, n) for fid in FIDS] + list(SPECIAL_SIZES[split].items()) |
| for k, (fid, count) in enumerate(plan): |
| for i in range(count): |
| seed = base + k * 1000 + i |
| row, why = operator_row(fid, seed, "ar" if i % 2 else "en", teacher) |
| if row: |
| rows.append(row) |
| else: |
| drops[f"operator {fid}: {why}"] += 1 |
| return rows |
|
|
|
|
| |
| FALLBACK_MODES = ["wrong", "policy", "ignored_error", "correct"] |
|
|
|
|
| def build_guardian(split, gb, drops): |
| rows, base, n = [], SEEDS["guardian"][split], SIZES["guardian"][split] |
| for k, fid in enumerate(FIDS): |
| for i in range(n): |
| seed = base + k * 1000 + i |
| lang = "ar" if i % 2 else "en" |
| mode = MODE_CYCLE[(i * 7 + k) % len(MODE_CYCLE)] |
| row = gb.build(fid, seed, lang, mode) |
| for alt in FALLBACK_MODES: |
| if row: |
| break |
| drops[f"guardian {fid}: {mode} -> {alt}"] += 1 |
| row = gb.build(fid, seed, lang, alt) |
| if row: |
| rows.append(row) |
| return rows |
|
|
|
|
| def build_verifier(split, vb): |
| rows, base, n = [], SEEDS["verifier"][split], SIZES["verifier"][split] |
| for k, fid in enumerate(FIDS): |
| for i in range(n): |
| seed = base + k * 1000 + i |
| case = CASE_CYCLE[(i * 4 + k) % len(CASE_CYCLE)] |
| rows.append(vb.build(fid, seed, "ar" if (i + k) % 2 else "en", case)) |
| return rows |
|
|
|
|
| |
| def build_evals(teacher, gb, vb): |
| ev_op = [] |
| order = FIDS * 2 + random.Random(7).sample(FIDS, 16) + ["OK00"] * 10 + ["UNK00"] * 10 |
| for j, fid in enumerate(order): |
| seed = SEEDS["operator"]["eval"] + j |
| lang = "ar" if j % 2 else "en" |
| ep = Simulator(seed).new_episode(fid, lang=lang) |
| s = teacher.run(Simulator(seed).new_episode(fid, lang=lang), random.Random(seed)) |
| ev_op.append({"id": f"op-{j:03d}", "fault_id": fid, "seed": seed, "lang": lang, |
| "messages": ep.display(s["messages"][:2]), "tools": s["tools"], |
| "ground_truth": ep.display(ep.ground_truth()), |
| "rebuild": f"Simulator({seed}).new_episode('{fid}', lang='{lang}')"}) |
| ev_g = [] |
| for k, fid in enumerate(FIDS): |
| for i, mode in enumerate(EVAL_MODES): |
| seed = SEEDS["guardian"]["eval"] + k * 1000 + i |
| row = gb.build(fid, seed, "ar" if i % 2 else "en", mode) |
| for alt in FALLBACK_MODES: |
| if row: |
| break |
| row = gb.build(fid, seed, "ar" if i % 2 else "en", alt) |
| if row: |
| ev_g.append({"id": f"g-{len(ev_g):03d}", "fault_id": fid, "seed": seed, |
| "lang": row["meta"]["lang"], "mode": row["meta"]["mode"], |
| "messages": row["messages"][:2], "tools": row["tools"], |
| "expected": json.loads(row["messages"][-1]["content"]), "gold_messages": row["messages"], |
| "meta": row["meta"]}) |
| ev_v = [] |
| for k, fid in enumerate(FIDS): |
| for i, case in enumerate(V_EVAL_CASES): |
| seed = SEEDS["verifier"]["eval"] + k * 1000 + i |
| lang = "ar" if (i + k) % 2 else "en" |
| row = vb.build(fid, seed, lang, case) |
| if row["meta"]["case"] != case: |
| continue |
| ev_v.append({"id": f"v-{len(ev_v):03d}", "fault_id": fid, "seed": seed, "lang": lang, "case": case, |
| "messages": row["messages"][:2], "tools": row["tools"], |
| "expected": json.loads(row["messages"][-1]["content"]), "gold_messages": row["messages"], |
| "meta": row["meta"]}) |
| return ev_op, ev_g, ev_v |
|
|
|
|
| |
| def tok_est(row) -> int: |
| return int(len(json.dumps(row["messages"], ensure_ascii=False)) / 3.2 |
| + len(json.dumps(row["tools"])) / 3.6) |
|
|
|
|
| def diversity(rows): |
| users = [r["messages"][1]["content"] for r in rows] |
| finals = [r["messages"][-1]["content"] for r in rows] |
| return {"unique_requests_pct": round(100 * len(set(users)) / len(users), 1), |
| "unique_final_answers_pct": round(100 * len(set(finals)) / len(finals), 1)} |
|
|
|
|
| def write(path: Path, rows): |
| with open(path, "w", encoding="utf-8") as f: |
| for r in rows: |
| f.write(json.dumps(r, ensure_ascii=False) + "\n") |
|
|
|
|
| def main(): |
| out = ROOT / "data" |
| for d in ("operator", "guardian", "verifier"): |
| (out / d).mkdir(parents=True, exist_ok=True) |
| teacher, gb, vb = ScriptedTeacher(), GuardianBuilder(), VerifierBuilder() |
| stats, drops = {}, Counter() |
| allrows = {} |
| for split in ("train", "val"): |
| op = build_operator(split, teacher, drops) |
| g = build_guardian(split, gb, drops) |
| v = build_verifier(split, vb) |
| for name, rows in (("operator", op), ("guardian", g), ("verifier", v)): |
| write(out / name / f"{name}_{split}.jsonl", rows) |
| toks = [tok_est(r) for r in rows] |
| labels = Counter(r["meta"].get("label") or r["meta"]["fault_id"][:1] for r in rows) |
| stats[f"{name}_{split}"] = { |
| "rows": len(rows), "arabic_pct": round(100 * sum(r["meta"]["lang"] == "ar" for r in rows) / len(rows), 1), |
| "labels": dict(labels) if name != "operator" else None, |
| "tokens_avg": int(sum(toks) / len(toks)), "tokens_max": max(toks), "tokens_total": sum(toks), |
| **diversity(rows)} |
| allrows[split] = op + g + v |
| print(split, {k: v["rows"] for k, v in stats.items() if k.endswith(split)}) |
|
|
| ev_op, ev_g, ev_v = build_evals(teacher, gb, vb) |
| write(out / "operator" / "operator_eval.jsonl", ev_op) |
| write(out / "guardian" / "guardian_eval.jsonl", ev_g) |
| write(out / "verifier" / "verifier_eval.jsonl", ev_v) |
| stats["eval"] = {"operator": len(ev_op), "guardian": len(ev_g), "verifier": len(ev_v), |
| "guardian_labels": dict(Counter(e["expected"]["decision"] for e in ev_g)), |
| "verifier_labels": dict(Counter(e["expected"]["status"] for e in ev_v))} |
| stats["dropped_or_replaced"] = dict(drops.most_common(25)) |
| stats["train_total_rows"] = len(allrows["train"]) |
| stats["train_total_tokens_est"] = sum(tok_est(r) for r in allrows["train"]) |
| leak = sum("__BADVM__" in json.dumps(r, ensure_ascii=False) for r in allrows["train"]) |
| stats["placeholder_leaks"] = leak |
| (out / "stats.json").write_text(json.dumps(stats, indent=2, ensure_ascii=False)) |
| print(json.dumps({k: stats[k] for k in ("eval", "train_total_rows", "train_total_tokens_est", |
| "placeholder_leaks")}, ensure_ascii=False)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
