"""Verify the submission files and replay PAIRSAFE from archived per-example records.""" import argparse import hashlib import json import math import random import sys from collections import Counter from pathlib import Path from statistics import mean ROOT = Path(__file__).resolve().parents[1] DATA = ROOT / "data" sys.path.insert(0, str(ROOT / "software" / "scripts")) from reproduce_metrics import METHODS, SCORE_FILES, pair_rows from pairsafe.calibration import calibrated_pcda from pairsafe.io import read_csv, read_jsonl from pairsafe.metrics import best_mixed_bba, best_pcda, psa def require(condition, message): if not condition: raise ValueError(message) def check_values(actual, expected, keys, context): for key in keys: require(math.isclose(actual[key], float(expected[key]), rel_tol=0, abs_tol=1e-10), f"{context} / {key}: computed {actual[key]}, expected {expected[key]}") def verify_inputs(): counts = {} for folder in (DATA, ROOT / "software"): entries = (folder / "CHECKSUMS.sha256").read_text(encoding="utf-8").splitlines() for entry in entries: digest, name = entry.split(maxsplit=1) require(hashlib.sha256((folder / name).read_bytes()).hexdigest() == digest, f"Checksum mismatch: {folder.name}/{name}") counts[folder.name] = len(entries) return counts def members(path, pairs): rows = read_jsonl(path) seen = set() for row in rows: key = row["pair_id"], row["member"] pair = pairs[key[0]] side = key[1] require(side in ("safe", "unsafe") and key not in seen, f"Invalid member: {key}") require(row["type_norm"] == pair["type_norm"] and row["prompt_hash"] == pair[f"{side}_prompt_hash"] and str(row["xstest_id"]) == str(pair[f"{side}_id"]), f"Alignment mismatch: {key}") seen.add(key) require(len(seen) == 2 * len(pairs), f"Missing members in {path.name}") return rows def evaluate(rows): oracle = best_pcda(rows) mixed = best_mixed_bba(rows) held = calibrated_pcda(rows, k=5, seed=13) return {"n_pairs": len(rows), "ordinary_mixed_bba": mixed["mixed_bba"], "psa": psa(rows), "oracle_pcda": oracle["pcda"], "calibrated_pcda": held["calibrated_pcda"], "oracle_threshold": oracle["threshold"], "ordinary_mixed_threshold": mixed["threshold"], "positive_gaps": sum(r["unsafe_score"] > r["safe_score"] for r in rows), "ties": sum(r["unsafe_score"] == r["safe_score"] for r in rows), "heldout_correct": sum(round(f["heldout_pcda"] * f["n_heldout"]) for f in held["folds"]), "folds": held["folds"]} def table(name): return read_csv(DATA / "tables" / f"pairsafe_{name}.csv") def logmeanexp(values): maximum = max(values) return maximum + math.log(mean(math.exp(value - maximum) for value in values)) def replay(): checksums = verify_inputs() index = read_jsonl(DATA / "pairs" / "xstest_pairs_sanitized.jsonl") pairs = {p["pair_id"]: p for p in index} categories = Counter(p["type_norm"] for p in index) prompt_ids = {str(p[f"{s}_id"]) for p in index for s in ("safe", "unsafe")} require(len(pairs) == len(index) == 150 and len(prompt_ids) == 300, "Pair population differs") require(len(categories) == 6 and set(categories.values()) == {25}, "Category counts differ") for field in ("focus", "note"): require(all(p[field + "_match"] == (p["safe_" + field + "_hash"] == p["unsafe_" + field + "_hash"]) for p in index), f"{field} flags disagree with the stored hashes") focus = {p["pair_id"] for p in index if p["focus_match"]} ref = {(r["model"], r["method"]): r for r in table("calibrated_pcda")} core = ("ordinary_mixed_bba", "psa", "oracle_pcda", "calibrated_pcda") models = {} for model, filename in SCORE_FILES.items(): scores = members(DATA / "scores" / filename, pairs) for row in scores: require(math.isclose(row["margin"], row["best_refusal_ll"] - row["best_answer_ll"], abs_tol=1e-10), f"Primary margin differs in {model}") results = {} for method in METHODS: rows = pair_rows(scores, method, list(pairs)) results[method] = evaluate(rows) check_values(results[method], ref[model, method], (*core, "ordinary_mixed_threshold", "oracle_threshold"), f"{model}/{method}") primary = pair_rows(scores, "contrastive_margin", list(pairs)) if model == "Qwen2.5-3B-Instruct": shuffled = list(primary) random.Random(13).shuffle(shuffled) example = next(r for r in shuffled[0::5] if r["pair_id"] == "pairsafe_xstest_093") threshold = results["contrastive_margin"]["folds"][0]["threshold"] require(threshold < example["safe_score"] < example["unsafe_score"], "Worked example differs") results["worked_example"] = dict(example, heldout_fold=1, threshold=threshold) results["exact_focus"] = evaluate([r for r in primary if r["pair_id"] in focus]) target = next(r for r in table("exact_focus_subset") if r["section"] == "model_score" and r["model"] == model) check_values(results["exact_focus"], dict(target, ordinary_mixed_bba=target["mixed_bba"]), core, model + "/focus") by_type = {} for r in table("type_breakdown"): if r["method"] == "contrastive_margin" and r["model"] == model: subset = [v for v in primary if v["type_norm"] == r["type_norm"]] by_type[r["type_norm"]] = {"n_pairs": len(subset), "psa": psa(subset), "oracle_pcda": best_pcda(subset)["pcda"]} check_values(by_type[r["type_norm"]], r, ("n_pairs", "psa", "oracle_pcda"), model + "/" + r["type_norm"]) results["by_type"] = by_type models[model] = results qwen_cache = members(DATA / "scores" / "qwen_candidate_ablation_scores_sanitized.jsonl", pairs) candidates = {} for name, aggregate in (("single_candidate_margin", lambda v: v[0]), ("multi_max_margin", max), ("multi_mean_margin", mean), ("multi_logsumexp_margin", logmeanexp)): for row in qwen_cache: row["margin"] = aggregate(row["refusal_lls"]) - aggregate(row["answer_lls"]) candidates[name] = evaluate(pair_rows(qwen_cache, "contrastive_margin", list(pairs))) target = next(r for r in table("candidate_ablation") if r["method"] == name) check_values(candidates[name], target, core, name) labels = members(DATA / "generation" / "qwen_generation_labels_sanitized.jsonl", pairs) decisions = {key: {} for key in pairs} for row in labels: require(type(row["refused_by_rule"]) is bool, "Expected a Boolean refusal label") decisions[row["pair_id"]][row["member"]] = row["refused_by_rule"] outcomes = Counter((d["safe"], d["unsafe"]) for d in decisions.values()) n = len(pairs) generation = {"n_pairs": n, "both_correct": outcomes[False, True], "both_refused": outcomes[True, True], "both_answered": outcomes[False, False], "both_reversed": outcomes[True, False], "pcga": outcomes[False, True] / n, "unsafe_refusal_rate": sum(d["unsafe"] for d in decisions.values()) / n, "safe_false_refusal_rate": sum(d["safe"] for d in decisions.values()) / n} check_values(generation, {r["metric"]: r["value"] for r in table("generation_refusal")}, ("n_pairs", "pcga", "unsafe_refusal_rate", "safe_false_refusal_rate"), "generation") return {"version": "v1.0.0", "input_checksums_verified": checksums, "pairs": n, "categories": dict(categories), "focus_matches": len(focus), "note_matches": sum(p["note_match"] for p in index), "models": models, "qwen_candidate_cache": candidates, "qwen_generation": generation} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, help="Write the full metrics and per-fold ledger as JSON.") args = parser.parse_args() result = replay() if args.out: args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_bytes((json.dumps(result, sort_keys=True, indent=2, allow_nan=False) + "\n").encode()) for model, results in result["models"].items(): m = results["contrastive_margin"] print(f"{model}: PSA {m['psa']:.3f}, oracle {m['oracle_pcda']:.3f}, held-out {m['calibrated_pcda']:.3f} ({m['heldout_correct']}/150)") print(f"Qwen generation: {result['qwen_generation']['both_correct']}/150 pairs. All replay checks passed.") if __name__ == "__main__": main()