v802-classifier / src /evaluate_v801.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
8.09 kB
#!/usr/bin/env python3
"""Evaluate v801 once on the locked new and historical manifests."""
from __future__ import annotations
import argparse
import csv
import json
from pathlib import Path
import joblib
import numpy as np
from scipy.stats import binomtest
from sklearn.metrics import accuracy_score, f1_score
from compare_checkpoints import sha256
from train_probe import apply_thresholds, metrics as legacy_metrics, predict as legacy_predict
from train_v801 import artifact_probabilities, report_metrics, structured_predictions
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--legacy-features", type=Path, required=True)
parser.add_argument("--before", type=Path, required=True)
parser.add_argument("--after", type=Path, required=True)
parser.add_argument("--new-holdout", type=Path, required=True)
parser.add_argument("--historical-holdout", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
rich = np.load(args.features)
legacy = np.load(args.legacy_features)
if not np.array_equal(rich["file_names"], legacy["file_names"]):
raise ValueError("rich/legacy feature filenames differ")
if not np.array_equal(rich["labels"], legacy["labels"]):
raise ValueError("rich/legacy labels differ")
if not np.array_equal(rich["classes"], legacy["classes"]):
raise ValueError("rich/legacy class order differs")
reconstruction_error = float(np.max(np.abs(rich["features"] - legacy["features"])))
if reconstruction_error > 2e-6:
raise ValueError(f"legacy feature reconstruction error is too large: {reconstruction_error}")
names = rich["file_names"].astype(str)
y = rich["labels"].astype(np.uint8)
classes = rich["classes"].astype(str)
name_to_index = {name: index for index, name in enumerate(names)}
before = joblib.load(args.before)
after = joblib.load(args.after)
if [str(value) for value in before["classes"]] != classes.tolist():
raise ValueError("v800 class order mismatch")
if [str(value) for value in after["classes"]] != classes.tolist():
raise ValueError("v801 class order mismatch")
if before["feature_file_sha256"] != sha256(args.legacy_features):
raise ValueError("v800 legacy feature provenance mismatch")
if after["feature_file_sha256"] != sha256(args.features):
raise ValueError("v801 rich feature provenance mismatch")
before_all_probabilities = legacy_predict(before, legacy["features"])
after_all_probabilities = artifact_probabilities(
after, rich["view_descriptors"], rich["features"]
)
def compare(manifest: Path) -> dict[str, object]:
with manifest.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
targets = y[indices]
before_probabilities = before_all_probabilities[indices]
after_probabilities = after_all_probabilities[indices]
before_predictions = apply_thresholds(
before_probabilities, np.asarray(before["thresholds"])
)
after_predictions = structured_predictions(after, after_probabilities)
outputs: dict[str, object] = {
"before_v800": legacy_metrics(
targets, before_probabilities, np.asarray(before["thresholds"])
),
"after_v801": report_metrics(targets, after_probabilities, after_predictions),
}
outputs["delta_v801_minus_v800"] = {
key: outputs["after_v801"][key] - outputs["before_v800"][key]
for key in (
"exact_match_accuracy", "micro_f1", "macro_f1",
"mean_average_precision", "macro_roc_auc", "hamming_loss",
)
}
before_exact = np.all(before_predictions == targets, axis=1)
after_exact = np.all(after_predictions == targets, axis=1)
improved = int(np.sum(~before_exact & after_exact))
regressed = int(np.sum(before_exact & ~after_exact))
outputs["paired_exact_match"] = {
"improved": improved,
"regressed": regressed,
"unchanged": int(np.sum(before_exact == after_exact)),
"exact_mcnemar_p_two_sided": (
float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue)
if improved + regressed else 1.0
),
}
rng = np.random.default_rng(801001)
bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")}
for _ in range(10000):
sample = rng.integers(0, len(indices), size=len(indices))
sampled_targets = targets[sample]
sampled_before = before_predictions[sample]
sampled_after = after_predictions[sample]
bootstrap["exact_match_accuracy"].append(
accuracy_score(sampled_targets, sampled_after)
- accuracy_score(sampled_targets, sampled_before)
)
for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")):
bootstrap[key].append(
f1_score(sampled_targets, sampled_after, average=average, zero_division=0)
- f1_score(sampled_targets, sampled_before, average=average, zero_division=0)
)
outputs["paired_bootstrap_delta_95pct"] = {
key: {
"low": float(np.quantile(values, 0.025)),
"high": float(np.quantile(values, 0.975)),
"resamples": 10000,
"seed": 801001,
}
for key, values in bootstrap.items()
}
changed = []
for local, row in enumerate(rows):
if not np.array_equal(before_predictions[local], after_predictions[local]):
changed.append({
"file_name": row["file_name"],
"true": classes[targets[local].astype(bool)].tolist(),
"v800": classes[before_predictions[local].astype(bool)].tolist(),
"v801": classes[after_predictions[local].astype(bool)].tolist(),
"v800_exact": bool(before_exact[local]),
"v801_exact": bool(after_exact[local]),
})
outputs["changed_predictions"] = changed
outputs["samples"] = len(indices)
outputs["file_names"] = [row["file_name"] for row in rows]
return outputs
report = {
"protocol": (
"The final local v801 was fitted using development736, with its fixed decoder "
"chosen using development OOF robustness and historical_holdout32 as a non-independent "
"reference. new_holdout32 was not used to configure the local model. Because an earlier "
"stage-1 v801 was already evaluated on new_holdout32, this is a post-evaluation recheck "
"rather than a pristine first-look test."
),
"features": {
"rich_path": str(args.features.resolve()),
"rich_sha256": sha256(args.features),
"legacy_path": str(args.legacy_features.resolve()),
"legacy_sha256": sha256(args.legacy_features),
"max_legacy_reconstruction_abs_error": reconstruction_error,
},
"checkpoints": {
"v800": {"path": str(args.before.resolve()), "sha256": sha256(args.before)},
"v801": {"path": str(args.after.resolve()), "sha256": sha256(args.after)},
},
"new_holdout32": compare(args.new_holdout),
"historical_holdout32": compare(args.historical_holdout),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()