"""Build every dataset Sanad needs, in one go. python scripts/make_all_data.py data/ operator/ operator_train.jsonl operator_val.jsonl operator_eval.jsonl (120 held-out scenarios) guardian/ guardian_train.jsonl guardian_val.jsonl guardian_eval.jsonl (252 review packets + expected decision) verifier/ verifier_train.jsonl verifier_val.jsonl verifier_eval.jsonl (re-checks + expected status) stats.json counts, label balance, diversity and length Each agent is trained separately (its own LoRA adapter on the same base model). Every row: {"messages": [...], "tools": [...], "meta": {"role": ..., ...}} Names of hosts, VMs, datastores, cluster and vCenter change in every episode. """ from __future__ import annotations import json import random import sys from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from sanad_sim import FAULTS, Simulator # noqa: E402 from sanad_sim.checks import check_grounding # noqa: E402 from sanad_sim.guardian_data import EVAL_MODES, MODE_CYCLE, GuardianBuilder # noqa: E402 from sanad_sim.teacher import ScriptedTeacher # noqa: E402 from sanad_sim.verifier_data import CASE_CYCLE, CASES as V_EVAL_CASES, VerifierBuilder # noqa: E402 FIDS = list(FAULTS) SEEDS = { # role -> split -> base seed (ranges never overlap) "operator": {"train": 0, "val": 500_000, "eval": 900_000}, "guardian": {"train": 1_100_000, "val": 1_500_000, "eval": 1_900_000}, "verifier": {"train": 2_100_000, "val": 2_500_000, "eval": 2_900_000}, } SIZES = {"operator": {"train": 70, "val": 8}, "guardian": {"train": 80, "val": 10}, "verifier": {"train": 40, "val": 5}} SPECIAL_SIZES = {"train": {"OK00": 150, "UNK00": 100}, "val": {"OK00": 20, "UNK00": 15}} P_TOOL_ERROR = 0.15 # operator def operator_row(fid, seed, lang, teacher): rng = random.Random(seed * 7919 + 13) probe = Simulator(seed).new_episode(fid, lang=lang) errs = None if fid not in ("UNK00",) and rng.random() < P_TOOL_ERROR: errs = [{"tool": probe.inj.evidence[0][0], "call": 1, "error": rng.choice(["timeout after 30s", "connection reset by vCenter", "503 Service Unavailable"])}] ep = Simulator(seed).new_episode(fid, lang=lang, tool_errors=errs) s = teacher.run(ep, rng) fa = s["first_action"] if fid == "UNK00": if check_grounding(s["final_answer"], ep.trace): return None, "grounding" else: chk = ep.code_checks(s["final_answer"], fa and fa["action"], fa and fa["args"]) if not chk["pass"]: return None, f"checks {[i['type'] for i in chk['issues']]}" for action, args in ep.inj.solution: ep.execute(action, args) if not ep.resolved(): return None, "fix did not resolve" msgs = ep.display(s["messages"]) # re-check grounding on what the model will actually see (after renaming) shown = [{"output": json.loads(m["content"])} for m in msgs if m["role"] == "tool"] shown += [{"tool": "propose_action", "args": tc["function"]["arguments"], "output": {}} for m in msgs if m["role"] == "assistant" for tc in m.get("tool_calls", []) if tc["function"]["name"] == "propose_action"] if fid != "UNK00" and check_grounding(msgs[-1]["content"], shown): return None, "grounding after rename" meta = {"role": "operator", "fault_id": fid, "seed": seed, "lang": lang, "tool_error": bool(errs), "ground_truth": ep.display(ep.ground_truth())} return {"messages": msgs, "tools": s["tools"], "meta": meta}, None def build_operator(split, teacher, drops): rows, base, n = [], SEEDS["operator"][split], SIZES["operator"][split] plan = [(fid, n) for fid in FIDS] + list(SPECIAL_SIZES[split].items()) for k, (fid, count) in enumerate(plan): for i in range(count): seed = base + k * 1000 + i row, why = operator_row(fid, seed, "ar" if i % 2 else "en", teacher) if row: rows.append(row) else: drops[f"operator {fid}: {why}"] += 1 return rows # guardian / verifier FALLBACK_MODES = ["wrong", "policy", "ignored_error", "correct"] def build_guardian(split, gb, drops): rows, base, n = [], SEEDS["guardian"][split], SIZES["guardian"][split] for k, fid in enumerate(FIDS): for i in range(n): seed = base + k * 1000 + i lang = "ar" if i % 2 else "en" mode = MODE_CYCLE[(i * 7 + k) % len(MODE_CYCLE)] # spread modes evenly in every split row = gb.build(fid, seed, lang, mode) for alt in FALLBACK_MODES: if row: break drops[f"guardian {fid}: {mode} -> {alt}"] += 1 row = gb.build(fid, seed, lang, alt) if row: rows.append(row) return rows def build_verifier(split, vb): rows, base, n = [], SEEDS["verifier"][split], SIZES["verifier"][split] for k, fid in enumerate(FIDS): for i in range(n): seed = base + k * 1000 + i case = CASE_CYCLE[(i * 4 + k) % len(CASE_CYCLE)] rows.append(vb.build(fid, seed, "ar" if (i + k) % 2 else "en", case)) return rows # eval sets def build_evals(teacher, gb, vb): ev_op = [] order = FIDS * 2 + random.Random(7).sample(FIDS, 16) + ["OK00"] * 10 + ["UNK00"] * 10 for j, fid in enumerate(order): seed = SEEDS["operator"]["eval"] + j lang = "ar" if j % 2 else "en" ep = Simulator(seed).new_episode(fid, lang=lang) s = teacher.run(Simulator(seed).new_episode(fid, lang=lang), random.Random(seed)) ev_op.append({"id": f"op-{j:03d}", "fault_id": fid, "seed": seed, "lang": lang, "messages": ep.display(s["messages"][:2]), "tools": s["tools"], "ground_truth": ep.display(ep.ground_truth()), "rebuild": f"Simulator({seed}).new_episode('{fid}', lang='{lang}')"}) ev_g = [] for k, fid in enumerate(FIDS): for i, mode in enumerate(EVAL_MODES): seed = SEEDS["guardian"]["eval"] + k * 1000 + i row = gb.build(fid, seed, "ar" if i % 2 else "en", mode) for alt in FALLBACK_MODES: if row: break row = gb.build(fid, seed, "ar" if i % 2 else "en", alt) if row: ev_g.append({"id": f"g-{len(ev_g):03d}", "fault_id": fid, "seed": seed, "lang": row["meta"]["lang"], "mode": row["meta"]["mode"], "messages": row["messages"][:2], "tools": row["tools"], "expected": json.loads(row["messages"][-1]["content"]), "gold_messages": row["messages"], "meta": row["meta"]}) ev_v = [] for k, fid in enumerate(FIDS): for i, case in enumerate(V_EVAL_CASES): seed = SEEDS["verifier"]["eval"] + k * 1000 + i lang = "ar" if (i + k) % 2 else "en" row = vb.build(fid, seed, lang, case) if row["meta"]["case"] != case: continue # this case does not exist for this fault ev_v.append({"id": f"v-{len(ev_v):03d}", "fault_id": fid, "seed": seed, "lang": lang, "case": case, "messages": row["messages"][:2], "tools": row["tools"], "expected": json.loads(row["messages"][-1]["content"]), "gold_messages": row["messages"], "meta": row["meta"]}) return ev_op, ev_g, ev_v # stats def tok_est(row) -> int: return int(len(json.dumps(row["messages"], ensure_ascii=False)) / 3.2 + len(json.dumps(row["tools"])) / 3.6) def diversity(rows): users = [r["messages"][1]["content"] for r in rows] finals = [r["messages"][-1]["content"] for r in rows] return {"unique_requests_pct": round(100 * len(set(users)) / len(users), 1), "unique_final_answers_pct": round(100 * len(set(finals)) / len(finals), 1)} def write(path: Path, rows): with open(path, "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n") def main(): out = ROOT / "data" for d in ("operator", "guardian", "verifier"): (out / d).mkdir(parents=True, exist_ok=True) teacher, gb, vb = ScriptedTeacher(), GuardianBuilder(), VerifierBuilder() stats, drops = {}, Counter() allrows = {} for split in ("train", "val"): op = build_operator(split, teacher, drops) g = build_guardian(split, gb, drops) v = build_verifier(split, vb) for name, rows in (("operator", op), ("guardian", g), ("verifier", v)): write(out / name / f"{name}_{split}.jsonl", rows) toks = [tok_est(r) for r in rows] labels = Counter(r["meta"].get("label") or r["meta"]["fault_id"][:1] for r in rows) stats[f"{name}_{split}"] = { "rows": len(rows), "arabic_pct": round(100 * sum(r["meta"]["lang"] == "ar" for r in rows) / len(rows), 1), "labels": dict(labels) if name != "operator" else None, "tokens_avg": int(sum(toks) / len(toks)), "tokens_max": max(toks), "tokens_total": sum(toks), **diversity(rows)} allrows[split] = op + g + v print(split, {k: v["rows"] for k, v in stats.items() if k.endswith(split)}) ev_op, ev_g, ev_v = build_evals(teacher, gb, vb) write(out / "operator" / "operator_eval.jsonl", ev_op) write(out / "guardian" / "guardian_eval.jsonl", ev_g) write(out / "verifier" / "verifier_eval.jsonl", ev_v) stats["eval"] = {"operator": len(ev_op), "guardian": len(ev_g), "verifier": len(ev_v), "guardian_labels": dict(Counter(e["expected"]["decision"] for e in ev_g)), "verifier_labels": dict(Counter(e["expected"]["status"] for e in ev_v))} stats["dropped_or_replaced"] = dict(drops.most_common(25)) stats["train_total_rows"] = len(allrows["train"]) stats["train_total_tokens_est"] = sum(tok_est(r) for r in allrows["train"]) leak = sum("__BADVM__" in json.dumps(r, ensure_ascii=False) for r in allrows["train"]) stats["placeholder_leaks"] = leak (out / "stats.json").write_text(json.dumps(stats, indent=2, ensure_ascii=False)) print(json.dumps({k: stats[k] for k in ("eval", "train_total_rows", "train_total_tokens_est", "placeholder_leaks")}, ensure_ascii=False)) if __name__ == "__main__": main()