v802-classifier / src /compare_checkpoints.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
6.09 kB
#!/usr/bin/env python3
"""Compare frozen-feature classifier checkpoints on identical locked manifests."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import joblib
import numpy as np
from scipy.stats import binomtest
from sklearn.metrics import accuracy_score, f1_score
from train_probe import apply_thresholds, metrics, predict
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--before", type=Path, required=True)
parser.add_argument("--after", type=Path, required=True)
parser.add_argument("--holdout", type=Path, required=True)
parser.add_argument("--historical-holdout", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
bundle = np.load(args.features)
feature_metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8"))
names = bundle["file_names"]
classes = [str(value) for value in bundle["classes"]]
x = bundle["features"]
y = bundle["labels"]
name_to_index = {str(name): index for index, name in enumerate(names)}
checkpoints = {"before": joblib.load(args.before), "after": joblib.load(args.after)}
for label, artifact in checkpoints.items():
if [str(value) for value in artifact["classes"]] != classes:
raise ValueError(f"{label} class order mismatch")
if artifact["feature_type"] != feature_metadata["feature_type"]:
raise ValueError(f"{label} feature schema mismatch")
if artifact["feature_model_file_sha256"] != feature_metadata["model_file_sha256"]:
raise ValueError(f"{label} backbone digest mismatch")
if int(artifact["feature_dimension"]) != x.shape[1]:
raise ValueError(f"{label} feature dimension mismatch")
def compare(manifest: Path) -> dict[str, object]:
with manifest.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
outputs: dict[str, object] = {}
exact: dict[str, np.ndarray] = {}
predictions_by_checkpoint: dict[str, np.ndarray] = {}
for label, artifact in checkpoints.items():
probabilities = predict(artifact, x[indices])
outputs[label] = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"]))
predictions = apply_thresholds(probabilities, np.asarray(artifact["thresholds"]))
predictions_by_checkpoint[label] = predictions
exact[label] = np.all(predictions == y[indices], axis=1)
outputs["delta_after_minus_before"] = {
key: outputs["after"][key] - outputs["before"][key]
for key in ("exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc")
}
discordant = np.logical_xor(exact["before"], exact["after"])
improved = int(np.sum(~exact["before"] & exact["after"]))
regressed = int(np.sum(exact["before"] & ~exact["after"]))
outputs["paired_exact_match"] = {
"improved": improved,
"regressed": regressed,
"unchanged": int(np.sum(~discordant)),
"exact_mcnemar_p_two_sided": (
float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue)
if improved + regressed else 1.0
),
}
rng = np.random.default_rng(800001)
bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")}
targets = y[indices]
for _ in range(10000):
sample = rng.integers(0, len(indices), size=len(indices))
before_sample = predictions_by_checkpoint["before"][sample]
after_sample = predictions_by_checkpoint["after"][sample]
target_sample = targets[sample]
bootstrap["exact_match_accuracy"].append(
accuracy_score(target_sample, after_sample) - accuracy_score(target_sample, before_sample)
)
for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")):
bootstrap[key].append(
f1_score(target_sample, after_sample, average=average, zero_division=0)
- f1_score(target_sample, before_sample, average=average, zero_division=0)
)
outputs["paired_bootstrap_delta_95pct"] = {
key: {
"low": float(np.quantile(values, 0.025)),
"high": float(np.quantile(values, 0.975)),
"resamples": 10000,
"seed": 800001,
}
for key, values in bootstrap.items()
}
outputs["samples"] = len(indices)
outputs["file_names"] = [row["file_name"] for row in rows]
return outputs
report = {
"protocol": "Paired before/after evaluation on identical locked data; frozen backbone and feature schema verified by SHA-256.",
"features": str(args.features.resolve()),
"features_sha256": sha256(args.features),
"checkpoints": {
"before": {"path": str(args.before.resolve()), "sha256": sha256(args.before)},
"after": {"path": str(args.after.resolve()), "sha256": sha256(args.after)},
},
"new_holdout32": compare(args.holdout),
}
if args.historical_holdout:
report["historical_holdout32"] = compare(args.historical_holdout)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()