#!/usr/bin/env python3 """Compare frozen-feature classifier checkpoints on identical locked manifests.""" from __future__ import annotations import argparse import csv import hashlib import json from pathlib import Path import joblib import numpy as np from scipy.stats import binomtest from sklearn.metrics import accuracy_score, f1_score from train_probe import apply_thresholds, metrics, predict def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--before", type=Path, required=True) parser.add_argument("--after", type=Path, required=True) parser.add_argument("--holdout", type=Path, required=True) parser.add_argument("--historical-holdout", type=Path) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() bundle = np.load(args.features) feature_metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8")) names = bundle["file_names"] classes = [str(value) for value in bundle["classes"]] x = bundle["features"] y = bundle["labels"] name_to_index = {str(name): index for index, name in enumerate(names)} checkpoints = {"before": joblib.load(args.before), "after": joblib.load(args.after)} for label, artifact in checkpoints.items(): if [str(value) for value in artifact["classes"]] != classes: raise ValueError(f"{label} class order mismatch") if artifact["feature_type"] != feature_metadata["feature_type"]: raise ValueError(f"{label} feature schema mismatch") if artifact["feature_model_file_sha256"] != feature_metadata["model_file_sha256"]: raise ValueError(f"{label} backbone digest mismatch") if int(artifact["feature_dimension"]) != x.shape[1]: raise ValueError(f"{label} feature dimension mismatch") def compare(manifest: Path) -> dict[str, object]: with manifest.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) outputs: dict[str, object] = {} exact: dict[str, np.ndarray] = {} predictions_by_checkpoint: dict[str, np.ndarray] = {} for label, artifact in checkpoints.items(): probabilities = predict(artifact, x[indices]) outputs[label] = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"])) predictions = apply_thresholds(probabilities, np.asarray(artifact["thresholds"])) predictions_by_checkpoint[label] = predictions exact[label] = np.all(predictions == y[indices], axis=1) outputs["delta_after_minus_before"] = { key: outputs["after"][key] - outputs["before"][key] for key in ("exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc") } discordant = np.logical_xor(exact["before"], exact["after"]) improved = int(np.sum(~exact["before"] & exact["after"])) regressed = int(np.sum(exact["before"] & ~exact["after"])) outputs["paired_exact_match"] = { "improved": improved, "regressed": regressed, "unchanged": int(np.sum(~discordant)), "exact_mcnemar_p_two_sided": ( float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue) if improved + regressed else 1.0 ), } rng = np.random.default_rng(800001) bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")} targets = y[indices] for _ in range(10000): sample = rng.integers(0, len(indices), size=len(indices)) before_sample = predictions_by_checkpoint["before"][sample] after_sample = predictions_by_checkpoint["after"][sample] target_sample = targets[sample] bootstrap["exact_match_accuracy"].append( accuracy_score(target_sample, after_sample) - accuracy_score(target_sample, before_sample) ) for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")): bootstrap[key].append( f1_score(target_sample, after_sample, average=average, zero_division=0) - f1_score(target_sample, before_sample, average=average, zero_division=0) ) outputs["paired_bootstrap_delta_95pct"] = { key: { "low": float(np.quantile(values, 0.025)), "high": float(np.quantile(values, 0.975)), "resamples": 10000, "seed": 800001, } for key, values in bootstrap.items() } outputs["samples"] = len(indices) outputs["file_names"] = [row["file_name"] for row in rows] return outputs report = { "protocol": "Paired before/after evaluation on identical locked data; frozen backbone and feature schema verified by SHA-256.", "features": str(args.features.resolve()), "features_sha256": sha256(args.features), "checkpoints": { "before": {"path": str(args.before.resolve()), "sha256": sha256(args.before)}, "after": {"path": str(args.after.resolve()), "sha256": sha256(args.after)}, }, "new_holdout32": compare(args.holdout), } if args.historical_holdout: report["historical_holdout32"] = compare(args.historical_holdout) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps(report, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()