Download src/compare_checkpoints.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 6.09 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/compare_checkpoints.py
- Command line
-
hf download hf://sneo/v802-classifier/src/compare_checkpoints.py
-
curl -L -o compare_checkpoints.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/compare_checkpoints.py
6.09 kB
| #!/usr/bin/env python3 | |
| """Compare frozen-feature classifier checkpoints on identical locked manifests.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from scipy.stats import binomtest | |
| from sklearn.metrics import accuracy_score, f1_score | |
| from train_probe import apply_thresholds, metrics, predict | |
| def sha256(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--before", type=Path, required=True) | |
| parser.add_argument("--after", type=Path, required=True) | |
| parser.add_argument("--holdout", type=Path, required=True) | |
| parser.add_argument("--historical-holdout", type=Path) | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| bundle = np.load(args.features) | |
| feature_metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8")) | |
| names = bundle["file_names"] | |
| classes = [str(value) for value in bundle["classes"]] | |
| x = bundle["features"] | |
| y = bundle["labels"] | |
| name_to_index = {str(name): index for index, name in enumerate(names)} | |
| checkpoints = {"before": joblib.load(args.before), "after": joblib.load(args.after)} | |
| for label, artifact in checkpoints.items(): | |
| if [str(value) for value in artifact["classes"]] != classes: | |
| raise ValueError(f"{label} class order mismatch") | |
| if artifact["feature_type"] != feature_metadata["feature_type"]: | |
| raise ValueError(f"{label} feature schema mismatch") | |
| if artifact["feature_model_file_sha256"] != feature_metadata["model_file_sha256"]: | |
| raise ValueError(f"{label} backbone digest mismatch") | |
| if int(artifact["feature_dimension"]) != x.shape[1]: | |
| raise ValueError(f"{label} feature dimension mismatch") | |
| def compare(manifest: Path) -> dict[str, object]: | |
| with manifest.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) | |
| outputs: dict[str, object] = {} | |
| exact: dict[str, np.ndarray] = {} | |
| predictions_by_checkpoint: dict[str, np.ndarray] = {} | |
| for label, artifact in checkpoints.items(): | |
| probabilities = predict(artifact, x[indices]) | |
| outputs[label] = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"])) | |
| predictions = apply_thresholds(probabilities, np.asarray(artifact["thresholds"])) | |
| predictions_by_checkpoint[label] = predictions | |
| exact[label] = np.all(predictions == y[indices], axis=1) | |
| outputs["delta_after_minus_before"] = { | |
| key: outputs["after"][key] - outputs["before"][key] | |
| for key in ("exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc") | |
| } | |
| discordant = np.logical_xor(exact["before"], exact["after"]) | |
| improved = int(np.sum(~exact["before"] & exact["after"])) | |
| regressed = int(np.sum(exact["before"] & ~exact["after"])) | |
| outputs["paired_exact_match"] = { | |
| "improved": improved, | |
| "regressed": regressed, | |
| "unchanged": int(np.sum(~discordant)), | |
| "exact_mcnemar_p_two_sided": ( | |
| float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue) | |
| if improved + regressed else 1.0 | |
| ), | |
| } | |
| rng = np.random.default_rng(800001) | |
| bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")} | |
| targets = y[indices] | |
| for _ in range(10000): | |
| sample = rng.integers(0, len(indices), size=len(indices)) | |
| before_sample = predictions_by_checkpoint["before"][sample] | |
| after_sample = predictions_by_checkpoint["after"][sample] | |
| target_sample = targets[sample] | |
| bootstrap["exact_match_accuracy"].append( | |
| accuracy_score(target_sample, after_sample) - accuracy_score(target_sample, before_sample) | |
| ) | |
| for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")): | |
| bootstrap[key].append( | |
| f1_score(target_sample, after_sample, average=average, zero_division=0) | |
| - f1_score(target_sample, before_sample, average=average, zero_division=0) | |
| ) | |
| outputs["paired_bootstrap_delta_95pct"] = { | |
| key: { | |
| "low": float(np.quantile(values, 0.025)), | |
| "high": float(np.quantile(values, 0.975)), | |
| "resamples": 10000, | |
| "seed": 800001, | |
| } | |
| for key, values in bootstrap.items() | |
| } | |
| outputs["samples"] = len(indices) | |
| outputs["file_names"] = [row["file_name"] for row in rows] | |
| return outputs | |
| report = { | |
| "protocol": "Paired before/after evaluation on identical locked data; frozen backbone and feature schema verified by SHA-256.", | |
| "features": str(args.features.resolve()), | |
| "features_sha256": sha256(args.features), | |
| "checkpoints": { | |
| "before": {"path": str(args.before.resolve()), "sha256": sha256(args.before)}, | |
| "after": {"path": str(args.after.resolve()), "sha256": sha256(args.after)}, | |
| }, | |
| "new_holdout32": compare(args.holdout), | |
| } | |
| if args.historical_holdout: | |
| report["historical_holdout32"] = compare(args.historical_holdout) | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |