tracegrpo / scripts /assemble_sft_set.py
Dhruv1000's picture
Add files using upload-large-folder tool
867597e verified
Raw History Blame Contribute Delete
5.75 kB
"""Validate gold/silver provenance and build the label-balanced SFT JSONL."""
from __future__ import annotations
import json
import hashlib
from collections import Counter
from pathlib import Path
import pandas as pd
from scripts.generate_teacher_corpus import _write_json_atomic, _write_jsonl_atomic
from src.contracts.parser import parse_output
from src.sft.teacher_filter import filter_teacher_rollout
from src.sft.topup import assemble_balanced_sft_rows
from src.sft.silver_gate import enforce_silver_gate
def _rows(path: str) -> list[dict]:
return [json.loads(line) for line in Path(path).open() if line.strip()]
def main() -> None:
main_report = json.loads(Path("results/phase4/teacher_gen_full_report.json").read_text())
topup_report = json.loads(Path("results/phase4/topup_report.json").read_text())
if main_report["n_batches_complete"] != main_report["n_batches_total"] or not topup_report["complete"]:
raise RuntimeError("main teacher and top-up must finish before SFT assembly")
main_gold = _rows("data/processed/teacher_traces.jsonl")
topup = _rows("data/processed/teacher_topup_traces.jsonl")
supplemental_report_path = Path("results/phase4/missing_cwe_topup_report.json")
supplemental = []
if supplemental_report_path.exists():
supplemental_report = json.loads(supplemental_report_path.read_text())
if not supplemental_report["complete"]:
raise RuntimeError("missing-CWE supplement must finish before SFT assembly")
supplemental = _rows("data/processed/teacher_missing_cwe_topup_traces.jsonl")
if {r["sample_id"] for r in topup} & {r["sample_id"] for r in supplemental}:
raise ValueError("missing-CWE supplement overlaps original top-up")
topup += supplemental
topup_gold = [r for r in topup if not r["silver"]]
topup_silver = [r for r in topup if r["silver"]]
main_covered = {r["sample_id"] for r in main_gold}
if main_covered & {r["sample_id"] for r in topup}:
raise ValueError("top-up overlaps samples already covered by main teacher")
topup_ids = {r["sample_id"] for r in topup}
annotations = {a["sample_id"]: a for line in Path("data/processed/annotations.jsonl").open()
if (a := json.loads(line))["sample_id"] in topup_ids}
for row in topup:
parsed = parse_output(row["teacher_output"])
ann = annotations.get(row["sample_id"], {})
result = filter_teacher_rollout(parsed, "VULNERABLE", ann.get("cwe"),
set(ann.get("l2_lines", [])))
if row["silver"]:
if result.reason != "iou_below_threshold":
raise ValueError(f"silver row does not satisfy silver-only rule: {row['sample_id']}")
elif not result.accepted:
raise ValueError(f"top-up gold row fails unchanged filter: {row['sample_id']}")
assembled, composition = assemble_balanced_sft_rows(main_gold, topup_gold, topup_silver)
if not assembled:
raise ValueError("assembled SFT set is empty")
counts = Counter("silver" if r["silver"] else "gold" for r in assembled)
corpus = pd.read_parquet("data/processed/corpus.parquet", columns=["sample_id", "cwe"])
cwe_by_id = dict(zip(corpus.sample_id, corpus.cwe.fillna("UNKNOWN")))
by_label_cwe: dict[str, dict] = {}
for row in assembled:
key = f"{'VULNERABLE' if int(row['target']) else 'SAFE'}:{cwe_by_id[row['sample_id']]}"
entry = by_label_cwe.setdefault(key, {"gold_traces": 0, "silver_traces": 0, "sample_ids": set()})
entry["silver_traces" if row["silver"] else "gold_traces"] += 1
entry["sample_ids"].add(row["sample_id"])
for entry in by_label_cwe.values():
entry["samples_with_trace"] = len(entry.pop("sample_ids"))
composition.update({"gold_total": counts["gold"], "silver_total": counts["silver"],
"per_label_cwe": dict(sorted(by_label_cwe.items())),
"topup_report": "results/phase4/topup_report.json",
"missing_cwe_topup_report": str(supplemental_report_path)
if supplemental else None,
"missing_cwe_supplemental_gold": sum(not r["silver"] for r in supplemental),
"missing_cwe_supplemental_silver": sum(r["silver"] for r in supplemental),
"main_teacher_report": "results/phase4/teacher_final_analysis.json",
"input_rows": len(main_gold)+len(topup),
"silver_fraction_of_all_sft_rows": counts["silver"] / len(assembled),
"assembled": True})
manifest_path = Path("data/processed/sft_train_manifest.jsonl")
_write_jsonl_atomic(manifest_path, [
{"row_index": index, "sample_id": row["sample_id"],
"target": int(row["target"]), "silver": row["silver"],
"origin": row["origin"], "statement_iou": row.get("statement_iou")}
for index, row in enumerate(assembled)])
composition["sft_manifest"] = str(manifest_path)
composition["sft_manifest_sha256"] = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
composition["vulnerable_silver_fraction"] = composition["silver_fraction_of_vulnerable"]
_write_jsonl_atomic(Path("data/processed/sft_train.jsonl"), assembled)
_write_json_atomic(Path("results/phase4/sft_set_composition.json"), composition)
print(json.dumps(composition, indent=2), flush=True)
if not enforce_silver_gate(composition):
print("SFT_PIPELINE_AWAITING_USER: vulnerable silver fraction exceeds 0.50; "
"full SFT has not started", flush=True)
raise SystemExit(75)
if __name__ == "__main__":
main()