Download src/evaluate_v801.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 8.09 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/evaluate_v801.py
- Command line
-
hf download hf://sneo/v802-classifier/src/evaluate_v801.py
-
curl -L -o evaluate_v801.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/evaluate_v801.py
8.09 kB
| #!/usr/bin/env python3 | |
| """Evaluate v801 once on the locked new and historical manifests.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from scipy.stats import binomtest | |
| from sklearn.metrics import accuracy_score, f1_score | |
| from compare_checkpoints import sha256 | |
| from train_probe import apply_thresholds, metrics as legacy_metrics, predict as legacy_predict | |
| from train_v801 import artifact_probabilities, report_metrics, structured_predictions | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--legacy-features", type=Path, required=True) | |
| parser.add_argument("--before", type=Path, required=True) | |
| parser.add_argument("--after", type=Path, required=True) | |
| parser.add_argument("--new-holdout", type=Path, required=True) | |
| parser.add_argument("--historical-holdout", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| rich = np.load(args.features) | |
| legacy = np.load(args.legacy_features) | |
| if not np.array_equal(rich["file_names"], legacy["file_names"]): | |
| raise ValueError("rich/legacy feature filenames differ") | |
| if not np.array_equal(rich["labels"], legacy["labels"]): | |
| raise ValueError("rich/legacy labels differ") | |
| if not np.array_equal(rich["classes"], legacy["classes"]): | |
| raise ValueError("rich/legacy class order differs") | |
| reconstruction_error = float(np.max(np.abs(rich["features"] - legacy["features"]))) | |
| if reconstruction_error > 2e-6: | |
| raise ValueError(f"legacy feature reconstruction error is too large: {reconstruction_error}") | |
| names = rich["file_names"].astype(str) | |
| y = rich["labels"].astype(np.uint8) | |
| classes = rich["classes"].astype(str) | |
| name_to_index = {name: index for index, name in enumerate(names)} | |
| before = joblib.load(args.before) | |
| after = joblib.load(args.after) | |
| if [str(value) for value in before["classes"]] != classes.tolist(): | |
| raise ValueError("v800 class order mismatch") | |
| if [str(value) for value in after["classes"]] != classes.tolist(): | |
| raise ValueError("v801 class order mismatch") | |
| if before["feature_file_sha256"] != sha256(args.legacy_features): | |
| raise ValueError("v800 legacy feature provenance mismatch") | |
| if after["feature_file_sha256"] != sha256(args.features): | |
| raise ValueError("v801 rich feature provenance mismatch") | |
| before_all_probabilities = legacy_predict(before, legacy["features"]) | |
| after_all_probabilities = artifact_probabilities( | |
| after, rich["view_descriptors"], rich["features"] | |
| ) | |
| def compare(manifest: Path) -> dict[str, object]: | |
| with manifest.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) | |
| targets = y[indices] | |
| before_probabilities = before_all_probabilities[indices] | |
| after_probabilities = after_all_probabilities[indices] | |
| before_predictions = apply_thresholds( | |
| before_probabilities, np.asarray(before["thresholds"]) | |
| ) | |
| after_predictions = structured_predictions(after, after_probabilities) | |
| outputs: dict[str, object] = { | |
| "before_v800": legacy_metrics( | |
| targets, before_probabilities, np.asarray(before["thresholds"]) | |
| ), | |
| "after_v801": report_metrics(targets, after_probabilities, after_predictions), | |
| } | |
| outputs["delta_v801_minus_v800"] = { | |
| key: outputs["after_v801"][key] - outputs["before_v800"][key] | |
| for key in ( | |
| "exact_match_accuracy", "micro_f1", "macro_f1", | |
| "mean_average_precision", "macro_roc_auc", "hamming_loss", | |
| ) | |
| } | |
| before_exact = np.all(before_predictions == targets, axis=1) | |
| after_exact = np.all(after_predictions == targets, axis=1) | |
| improved = int(np.sum(~before_exact & after_exact)) | |
| regressed = int(np.sum(before_exact & ~after_exact)) | |
| outputs["paired_exact_match"] = { | |
| "improved": improved, | |
| "regressed": regressed, | |
| "unchanged": int(np.sum(before_exact == after_exact)), | |
| "exact_mcnemar_p_two_sided": ( | |
| float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue) | |
| if improved + regressed else 1.0 | |
| ), | |
| } | |
| rng = np.random.default_rng(801001) | |
| bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")} | |
| for _ in range(10000): | |
| sample = rng.integers(0, len(indices), size=len(indices)) | |
| sampled_targets = targets[sample] | |
| sampled_before = before_predictions[sample] | |
| sampled_after = after_predictions[sample] | |
| bootstrap["exact_match_accuracy"].append( | |
| accuracy_score(sampled_targets, sampled_after) | |
| - accuracy_score(sampled_targets, sampled_before) | |
| ) | |
| for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")): | |
| bootstrap[key].append( | |
| f1_score(sampled_targets, sampled_after, average=average, zero_division=0) | |
| - f1_score(sampled_targets, sampled_before, average=average, zero_division=0) | |
| ) | |
| outputs["paired_bootstrap_delta_95pct"] = { | |
| key: { | |
| "low": float(np.quantile(values, 0.025)), | |
| "high": float(np.quantile(values, 0.975)), | |
| "resamples": 10000, | |
| "seed": 801001, | |
| } | |
| for key, values in bootstrap.items() | |
| } | |
| changed = [] | |
| for local, row in enumerate(rows): | |
| if not np.array_equal(before_predictions[local], after_predictions[local]): | |
| changed.append({ | |
| "file_name": row["file_name"], | |
| "true": classes[targets[local].astype(bool)].tolist(), | |
| "v800": classes[before_predictions[local].astype(bool)].tolist(), | |
| "v801": classes[after_predictions[local].astype(bool)].tolist(), | |
| "v800_exact": bool(before_exact[local]), | |
| "v801_exact": bool(after_exact[local]), | |
| }) | |
| outputs["changed_predictions"] = changed | |
| outputs["samples"] = len(indices) | |
| outputs["file_names"] = [row["file_name"] for row in rows] | |
| return outputs | |
| report = { | |
| "protocol": ( | |
| "The final local v801 was fitted using development736, with its fixed decoder " | |
| "chosen using development OOF robustness and historical_holdout32 as a non-independent " | |
| "reference. new_holdout32 was not used to configure the local model. Because an earlier " | |
| "stage-1 v801 was already evaluated on new_holdout32, this is a post-evaluation recheck " | |
| "rather than a pristine first-look test." | |
| ), | |
| "features": { | |
| "rich_path": str(args.features.resolve()), | |
| "rich_sha256": sha256(args.features), | |
| "legacy_path": str(args.legacy_features.resolve()), | |
| "legacy_sha256": sha256(args.legacy_features), | |
| "max_legacy_reconstruction_abs_error": reconstruction_error, | |
| }, | |
| "checkpoints": { | |
| "v800": {"path": str(args.before.resolve()), "sha256": sha256(args.before)}, | |
| "v801": {"path": str(args.after.resolve()), "sha256": sha256(args.after)}, | |
| }, | |
| "new_holdout32": compare(args.new_holdout), | |
| "historical_holdout32": compare(args.historical_holdout), | |
| } | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |