"""Validate gold/silver provenance and build the label-balanced SFT JSONL.""" from __future__ import annotations import json import hashlib from collections import Counter from pathlib import Path import pandas as pd from scripts.generate_teacher_corpus import _write_json_atomic, _write_jsonl_atomic from src.contracts.parser import parse_output from src.sft.teacher_filter import filter_teacher_rollout from src.sft.topup import assemble_balanced_sft_rows from src.sft.silver_gate import enforce_silver_gate def _rows(path: str) -> list[dict]: return [json.loads(line) for line in Path(path).open() if line.strip()] def main() -> None: main_report = json.loads(Path("results/phase4/teacher_gen_full_report.json").read_text()) topup_report = json.loads(Path("results/phase4/topup_report.json").read_text()) if main_report["n_batches_complete"] != main_report["n_batches_total"] or not topup_report["complete"]: raise RuntimeError("main teacher and top-up must finish before SFT assembly") main_gold = _rows("data/processed/teacher_traces.jsonl") topup = _rows("data/processed/teacher_topup_traces.jsonl") supplemental_report_path = Path("results/phase4/missing_cwe_topup_report.json") supplemental = [] if supplemental_report_path.exists(): supplemental_report = json.loads(supplemental_report_path.read_text()) if not supplemental_report["complete"]: raise RuntimeError("missing-CWE supplement must finish before SFT assembly") supplemental = _rows("data/processed/teacher_missing_cwe_topup_traces.jsonl") if {r["sample_id"] for r in topup} & {r["sample_id"] for r in supplemental}: raise ValueError("missing-CWE supplement overlaps original top-up") topup += supplemental topup_gold = [r for r in topup if not r["silver"]] topup_silver = [r for r in topup if r["silver"]] main_covered = {r["sample_id"] for r in main_gold} if main_covered & {r["sample_id"] for r in topup}: raise ValueError("top-up overlaps samples already covered by main teacher") topup_ids = {r["sample_id"] for r in topup} annotations = {a["sample_id"]: a for line in Path("data/processed/annotations.jsonl").open() if (a := json.loads(line))["sample_id"] in topup_ids} for row in topup: parsed = parse_output(row["teacher_output"]) ann = annotations.get(row["sample_id"], {}) result = filter_teacher_rollout(parsed, "VULNERABLE", ann.get("cwe"), set(ann.get("l2_lines", []))) if row["silver"]: if result.reason != "iou_below_threshold": raise ValueError(f"silver row does not satisfy silver-only rule: {row['sample_id']}") elif not result.accepted: raise ValueError(f"top-up gold row fails unchanged filter: {row['sample_id']}") assembled, composition = assemble_balanced_sft_rows(main_gold, topup_gold, topup_silver) if not assembled: raise ValueError("assembled SFT set is empty") counts = Counter("silver" if r["silver"] else "gold" for r in assembled) corpus = pd.read_parquet("data/processed/corpus.parquet", columns=["sample_id", "cwe"]) cwe_by_id = dict(zip(corpus.sample_id, corpus.cwe.fillna("UNKNOWN"))) by_label_cwe: dict[str, dict] = {} for row in assembled: key = f"{'VULNERABLE' if int(row['target']) else 'SAFE'}:{cwe_by_id[row['sample_id']]}" entry = by_label_cwe.setdefault(key, {"gold_traces": 0, "silver_traces": 0, "sample_ids": set()}) entry["silver_traces" if row["silver"] else "gold_traces"] += 1 entry["sample_ids"].add(row["sample_id"]) for entry in by_label_cwe.values(): entry["samples_with_trace"] = len(entry.pop("sample_ids")) composition.update({"gold_total": counts["gold"], "silver_total": counts["silver"], "per_label_cwe": dict(sorted(by_label_cwe.items())), "topup_report": "results/phase4/topup_report.json", "missing_cwe_topup_report": str(supplemental_report_path) if supplemental else None, "missing_cwe_supplemental_gold": sum(not r["silver"] for r in supplemental), "missing_cwe_supplemental_silver": sum(r["silver"] for r in supplemental), "main_teacher_report": "results/phase4/teacher_final_analysis.json", "input_rows": len(main_gold)+len(topup), "silver_fraction_of_all_sft_rows": counts["silver"] / len(assembled), "assembled": True}) manifest_path = Path("data/processed/sft_train_manifest.jsonl") _write_jsonl_atomic(manifest_path, [ {"row_index": index, "sample_id": row["sample_id"], "target": int(row["target"]), "silver": row["silver"], "origin": row["origin"], "statement_iou": row.get("statement_iou")} for index, row in enumerate(assembled)]) composition["sft_manifest"] = str(manifest_path) composition["sft_manifest_sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() composition["vulnerable_silver_fraction"] = composition["silver_fraction_of_vulnerable"] _write_jsonl_atomic(Path("data/processed/sft_train.jsonl"), assembled) _write_json_atomic(Path("results/phase4/sft_set_composition.json"), composition) print(json.dumps(composition, indent=2), flush=True) if not enforce_silver_gate(composition): print("SFT_PIPELINE_AWAITING_USER: vulnerable silver fraction exceeds 0.50; " "full SFT has not started", flush=True) raise SystemExit(75) if __name__ == "__main__": main()