#!/usr/bin/env python3 """Evaluate v801 once on the locked new and historical manifests.""" from __future__ import annotations import argparse import csv import json from pathlib import Path import joblib import numpy as np from scipy.stats import binomtest from sklearn.metrics import accuracy_score, f1_score from compare_checkpoints import sha256 from train_probe import apply_thresholds, metrics as legacy_metrics, predict as legacy_predict from train_v801 import artifact_probabilities, report_metrics, structured_predictions def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--legacy-features", type=Path, required=True) parser.add_argument("--before", type=Path, required=True) parser.add_argument("--after", type=Path, required=True) parser.add_argument("--new-holdout", type=Path, required=True) parser.add_argument("--historical-holdout", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() rich = np.load(args.features) legacy = np.load(args.legacy_features) if not np.array_equal(rich["file_names"], legacy["file_names"]): raise ValueError("rich/legacy feature filenames differ") if not np.array_equal(rich["labels"], legacy["labels"]): raise ValueError("rich/legacy labels differ") if not np.array_equal(rich["classes"], legacy["classes"]): raise ValueError("rich/legacy class order differs") reconstruction_error = float(np.max(np.abs(rich["features"] - legacy["features"]))) if reconstruction_error > 2e-6: raise ValueError(f"legacy feature reconstruction error is too large: {reconstruction_error}") names = rich["file_names"].astype(str) y = rich["labels"].astype(np.uint8) classes = rich["classes"].astype(str) name_to_index = {name: index for index, name in enumerate(names)} before = joblib.load(args.before) after = joblib.load(args.after) if [str(value) for value in before["classes"]] != classes.tolist(): raise ValueError("v800 class order mismatch") if [str(value) for value in after["classes"]] != classes.tolist(): raise ValueError("v801 class order mismatch") if before["feature_file_sha256"] != sha256(args.legacy_features): raise ValueError("v800 legacy feature provenance mismatch") if after["feature_file_sha256"] != sha256(args.features): raise ValueError("v801 rich feature provenance mismatch") before_all_probabilities = legacy_predict(before, legacy["features"]) after_all_probabilities = artifact_probabilities( after, rich["view_descriptors"], rich["features"] ) def compare(manifest: Path) -> dict[str, object]: with manifest.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) targets = y[indices] before_probabilities = before_all_probabilities[indices] after_probabilities = after_all_probabilities[indices] before_predictions = apply_thresholds( before_probabilities, np.asarray(before["thresholds"]) ) after_predictions = structured_predictions(after, after_probabilities) outputs: dict[str, object] = { "before_v800": legacy_metrics( targets, before_probabilities, np.asarray(before["thresholds"]) ), "after_v801": report_metrics(targets, after_probabilities, after_predictions), } outputs["delta_v801_minus_v800"] = { key: outputs["after_v801"][key] - outputs["before_v800"][key] for key in ( "exact_match_accuracy", "micro_f1", "macro_f1", "mean_average_precision", "macro_roc_auc", "hamming_loss", ) } before_exact = np.all(before_predictions == targets, axis=1) after_exact = np.all(after_predictions == targets, axis=1) improved = int(np.sum(~before_exact & after_exact)) regressed = int(np.sum(before_exact & ~after_exact)) outputs["paired_exact_match"] = { "improved": improved, "regressed": regressed, "unchanged": int(np.sum(before_exact == after_exact)), "exact_mcnemar_p_two_sided": ( float(binomtest(min(improved, regressed), improved + regressed, 0.5).pvalue) if improved + regressed else 1.0 ), } rng = np.random.default_rng(801001) bootstrap = {name: [] for name in ("exact_match_accuracy", "micro_f1", "macro_f1")} for _ in range(10000): sample = rng.integers(0, len(indices), size=len(indices)) sampled_targets = targets[sample] sampled_before = before_predictions[sample] sampled_after = after_predictions[sample] bootstrap["exact_match_accuracy"].append( accuracy_score(sampled_targets, sampled_after) - accuracy_score(sampled_targets, sampled_before) ) for average, key in (("micro", "micro_f1"), ("macro", "macro_f1")): bootstrap[key].append( f1_score(sampled_targets, sampled_after, average=average, zero_division=0) - f1_score(sampled_targets, sampled_before, average=average, zero_division=0) ) outputs["paired_bootstrap_delta_95pct"] = { key: { "low": float(np.quantile(values, 0.025)), "high": float(np.quantile(values, 0.975)), "resamples": 10000, "seed": 801001, } for key, values in bootstrap.items() } changed = [] for local, row in enumerate(rows): if not np.array_equal(before_predictions[local], after_predictions[local]): changed.append({ "file_name": row["file_name"], "true": classes[targets[local].astype(bool)].tolist(), "v800": classes[before_predictions[local].astype(bool)].tolist(), "v801": classes[after_predictions[local].astype(bool)].tolist(), "v800_exact": bool(before_exact[local]), "v801_exact": bool(after_exact[local]), }) outputs["changed_predictions"] = changed outputs["samples"] = len(indices) outputs["file_names"] = [row["file_name"] for row in rows] return outputs report = { "protocol": ( "The final local v801 was fitted using development736, with its fixed decoder " "chosen using development OOF robustness and historical_holdout32 as a non-independent " "reference. new_holdout32 was not used to configure the local model. Because an earlier " "stage-1 v801 was already evaluated on new_holdout32, this is a post-evaluation recheck " "rather than a pristine first-look test." ), "features": { "rich_path": str(args.features.resolve()), "rich_sha256": sha256(args.features), "legacy_path": str(args.legacy_features.resolve()), "legacy_sha256": sha256(args.legacy_features), "max_legacy_reconstruction_abs_error": reconstruction_error, }, "checkpoints": { "v800": {"path": str(args.before.resolve()), "sha256": sha256(args.before)}, "v801": {"path": str(args.after.resolve()), "sha256": sha256(args.after)}, }, "new_holdout32": compare(args.new_holdout), "historical_holdout32": compare(args.historical_holdout), } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps(report, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()