Download scripts/assemble_sft_set.py from Dhruv1000/tracegrpo: direct link, hf CLI and curl.
- Browser
- Download file 5.75 kB
-
https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/assemble_sft_set.py
- Command line
-
hf download hf://Dhruv1000/tracegrpo/scripts/assemble_sft_set.py
-
curl -L -o assemble_sft_set.py https://huggingface.co/Dhruv1000/tracegrpo/resolve/main/scripts/assemble_sft_set.py
5.75 kB
| """Validate gold/silver provenance and build the label-balanced SFT JSONL.""" | |
| from __future__ import annotations | |
| import json | |
| import hashlib | |
| from collections import Counter | |
| from pathlib import Path | |
| import pandas as pd | |
| from scripts.generate_teacher_corpus import _write_json_atomic, _write_jsonl_atomic | |
| from src.contracts.parser import parse_output | |
| from src.sft.teacher_filter import filter_teacher_rollout | |
| from src.sft.topup import assemble_balanced_sft_rows | |
| from src.sft.silver_gate import enforce_silver_gate | |
| def _rows(path: str) -> list[dict]: | |
| return [json.loads(line) for line in Path(path).open() if line.strip()] | |
| def main() -> None: | |
| main_report = json.loads(Path("results/phase4/teacher_gen_full_report.json").read_text()) | |
| topup_report = json.loads(Path("results/phase4/topup_report.json").read_text()) | |
| if main_report["n_batches_complete"] != main_report["n_batches_total"] or not topup_report["complete"]: | |
| raise RuntimeError("main teacher and top-up must finish before SFT assembly") | |
| main_gold = _rows("data/processed/teacher_traces.jsonl") | |
| topup = _rows("data/processed/teacher_topup_traces.jsonl") | |
| supplemental_report_path = Path("results/phase4/missing_cwe_topup_report.json") | |
| supplemental = [] | |
| if supplemental_report_path.exists(): | |
| supplemental_report = json.loads(supplemental_report_path.read_text()) | |
| if not supplemental_report["complete"]: | |
| raise RuntimeError("missing-CWE supplement must finish before SFT assembly") | |
| supplemental = _rows("data/processed/teacher_missing_cwe_topup_traces.jsonl") | |
| if {r["sample_id"] for r in topup} & {r["sample_id"] for r in supplemental}: | |
| raise ValueError("missing-CWE supplement overlaps original top-up") | |
| topup += supplemental | |
| topup_gold = [r for r in topup if not r["silver"]] | |
| topup_silver = [r for r in topup if r["silver"]] | |
| main_covered = {r["sample_id"] for r in main_gold} | |
| if main_covered & {r["sample_id"] for r in topup}: | |
| raise ValueError("top-up overlaps samples already covered by main teacher") | |
| topup_ids = {r["sample_id"] for r in topup} | |
| annotations = {a["sample_id"]: a for line in Path("data/processed/annotations.jsonl").open() | |
| if (a := json.loads(line))["sample_id"] in topup_ids} | |
| for row in topup: | |
| parsed = parse_output(row["teacher_output"]) | |
| ann = annotations.get(row["sample_id"], {}) | |
| result = filter_teacher_rollout(parsed, "VULNERABLE", ann.get("cwe"), | |
| set(ann.get("l2_lines", []))) | |
| if row["silver"]: | |
| if result.reason != "iou_below_threshold": | |
| raise ValueError(f"silver row does not satisfy silver-only rule: {row['sample_id']}") | |
| elif not result.accepted: | |
| raise ValueError(f"top-up gold row fails unchanged filter: {row['sample_id']}") | |
| assembled, composition = assemble_balanced_sft_rows(main_gold, topup_gold, topup_silver) | |
| if not assembled: | |
| raise ValueError("assembled SFT set is empty") | |
| counts = Counter("silver" if r["silver"] else "gold" for r in assembled) | |
| corpus = pd.read_parquet("data/processed/corpus.parquet", columns=["sample_id", "cwe"]) | |
| cwe_by_id = dict(zip(corpus.sample_id, corpus.cwe.fillna("UNKNOWN"))) | |
| by_label_cwe: dict[str, dict] = {} | |
| for row in assembled: | |
| key = f"{'VULNERABLE' if int(row['target']) else 'SAFE'}:{cwe_by_id[row['sample_id']]}" | |
| entry = by_label_cwe.setdefault(key, {"gold_traces": 0, "silver_traces": 0, "sample_ids": set()}) | |
| entry["silver_traces" if row["silver"] else "gold_traces"] += 1 | |
| entry["sample_ids"].add(row["sample_id"]) | |
| for entry in by_label_cwe.values(): | |
| entry["samples_with_trace"] = len(entry.pop("sample_ids")) | |
| composition.update({"gold_total": counts["gold"], "silver_total": counts["silver"], | |
| "per_label_cwe": dict(sorted(by_label_cwe.items())), | |
| "topup_report": "results/phase4/topup_report.json", | |
| "missing_cwe_topup_report": str(supplemental_report_path) | |
| if supplemental else None, | |
| "missing_cwe_supplemental_gold": sum(not r["silver"] for r in supplemental), | |
| "missing_cwe_supplemental_silver": sum(r["silver"] for r in supplemental), | |
| "main_teacher_report": "results/phase4/teacher_final_analysis.json", | |
| "input_rows": len(main_gold)+len(topup), | |
| "silver_fraction_of_all_sft_rows": counts["silver"] / len(assembled), | |
| "assembled": True}) | |
| manifest_path = Path("data/processed/sft_train_manifest.jsonl") | |
| _write_jsonl_atomic(manifest_path, [ | |
| {"row_index": index, "sample_id": row["sample_id"], | |
| "target": int(row["target"]), "silver": row["silver"], | |
| "origin": row["origin"], "statement_iou": row.get("statement_iou")} | |
| for index, row in enumerate(assembled)]) | |
| composition["sft_manifest"] = str(manifest_path) | |
| composition["sft_manifest_sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest() | |
| composition["vulnerable_silver_fraction"] = composition["silver_fraction_of_vulnerable"] | |
| _write_jsonl_atomic(Path("data/processed/sft_train.jsonl"), assembled) | |
| _write_json_atomic(Path("results/phase4/sft_set_composition.json"), composition) | |
| print(json.dumps(composition, indent=2), flush=True) | |
| if not enforce_silver_gate(composition): | |
| print("SFT_PIPELINE_AWAITING_USER: vulnerable silver fraction exceeds 0.50; " | |
| "full SFT has not started", flush=True) | |
| raise SystemExit(75) | |
| if __name__ == "__main__": | |
| main() | |