File size: 6,085 Bytes
59839e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | #!/usr/bin/env python3
"""Compare frozen-feature classifier checkpoints on identical locked manifests."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import joblib
import numpy as np
from scipy.stats import binomtest
from sklearn.metrics import accuracy_score, f1_score
from train_probe import apply_thresholds, metrics, predict
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--before", type=Path, required=True)
parser.add_argument("--after", type=Path, required=True)
parser.add_argument("--holdout", type=Path, required=True)
parser.add_argument("--historical-holdout", type=Path)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
bundle = np.load(args.features)
feature_metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8"))
names = bundle["file_names"]
classes = [str(value) for value in bundle["classes"]]
x = bundle["features"]
y = bundle["labels"]
name_to_index = {str(name): index for index, name in enumerate(names)}
checkpoints = {"before": joblib.load(args.before), "after": joblib.load(args.after)}
for label, artifact in checkpoints.items():
if [str(value) for value in artifact["classes"]] != classes:
raise ValueError(f"{label} class order mismatch")
if artifact["feature_type"] != feature_metadata["feature_type"]:
raise ValueError(f"{label} feature schema mismatch")
if artifact["feature_model_file_sha256"] != feature_metadata["model_file_sha256"]:
raise ValueError(f"{label} backbone digest mismatch")
if int(artifact["feature_dimension"]) != x.shape[1]:
raise ValueError(f"{label} feature dimension mismatch")
def compare(manifest: Path) -> dict[str, object]:
with manifest.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
outputs: dict[str, object] = {}
exact: dict[str, np.ndarray] = {}
predictions_by_checkpoint: dict[str, np.ndarray] = {}
for label, artifact in checkpoints.items():
probabilities = predict(artifact, x[indices])
outputs[label] = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"]))
predictions = apply_thresholds(probabilities, np.asarray(artifact["thresholds"]))
predictions_by_checkpoint[label] = predictions
exact[label] = np.all(predictions == y[indices], axis=1)
outputs["delta_after_minus_before"] = {
key: outputs["after"][key] - outputs["before"][key]
for key in ("exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc")
}
discordant = np.logical_xor(exact["before"], exact["after"])
improved = int(np.sum(~exact["before"] & exact["after"]))
regressed = int(np.sum(exact["before"] & ~exact["after"]))
outputs["paired_exact_match"] = {
"improved": improved,
"regressed": regressed,
"unchanged": int(np.sum(~discordant)),
"exact_mcnemar_p_two_sided": (
float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue)
if improved + regressed else 1.0
),
}
rng = np.random.default_rng(800001)
bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")}
targets = y[indices]
for _ in range(10000):
sample = rng.integers(0, len(indices), size=len(indices))
before_sample = predictions_by_checkpoint["before"][sample]
after_sample = predictions_by_checkpoint["after"][sample]
target_sample = targets[sample]
bootstrap["exact_match_accuracy"].append(
accuracy_score(target_sample, after_sample) - accuracy_score(target_sample, before_sample)
)
for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")):
bootstrap[key].append(
f1_score(target_sample, after_sample, average=average, zero_division=0)
- f1_score(target_sample, before_sample, average=average, zero_division=0)
)
outputs["paired_bootstrap_delta_95pct"] = {
key: {
"low": float(np.quantile(values, 0.025)),
"high": float(np.quantile(values, 0.975)),
"resamples": 10000,
"seed": 800001,
}
for key, values in bootstrap.items()
}
outputs["samples"] = len(indices)
outputs["file_names"] = [row["file_name"] for row in rows]
return outputs
report = {
"protocol": "Paired before/after evaluation on identical locked data; frozen backbone and feature schema verified by SHA-256.",
"features": str(args.features.resolve()),
"features_sha256": sha256(args.features),
"checkpoints": {
"before": {"path": str(args.before.resolve()), "sha256": sha256(args.before)},
"after": {"path": str(args.after.resolve()), "sha256": sha256(args.after)},
},
"new_holdout32": compare(args.holdout),
}
if args.historical_holdout:
report["historical_holdout32"] = compare(args.historical_holdout)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
|