#!/usr/bin/env python3 """Evaluate a frozen DINOv3 probe on manifests excluded from model selection.""" from __future__ import annotations import argparse import csv import hashlib import json from pathlib import Path import joblib import numpy as np from train_probe import metrics, predict def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def evaluate_manifest( manifest: Path, names: np.ndarray, x: np.ndarray, y: np.ndarray, artifact: dict[str, object], classes: list[str], ) -> dict[str, object]: with manifest.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) name_to_index = {str(name): index for index, name in enumerate(names)} indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) probabilities = predict(artifact, x[indices]) result = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"])) result["samples"] = len(indices) result["file_names"] = [row["file_name"] for row in rows] result["classes"] = classes return result def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--classifier", type=Path, required=True) parser.add_argument("--new-holdout", type=Path, required=True) parser.add_argument("--historical-holdout", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() bundle = np.load(args.features) artifact = joblib.load(args.classifier) classes = [str(value) for value in bundle["classes"]] if classes != [str(value) for value in artifact["classes"]]: raise ValueError("feature/classifier class order mismatch") if int(bundle["features"].shape[1]) != int(artifact["feature_dimension"]): raise ValueError("feature/classifier dimension mismatch") if file_sha256(args.features) != artifact["feature_file_sha256"]: raise ValueError("feature file provenance mismatch") for model in artifact["models"]: if int(model.n_features_in_) != int(artifact["feature_dimension"]): raise ValueError("classifier coefficient dimension mismatch") report = { "protocol": ( "new_holdout20 is the primary untouched test; historical_holdout32 is reported " "only as a comparison with the already published v2 result" ), "selected_name": artifact["selected_name"], "new_holdout20": evaluate_manifest( args.new_holdout, bundle["file_names"], bundle["features"], bundle["labels"], artifact, classes, ), "historical_holdout32": evaluate_manifest( args.historical_holdout, bundle["file_names"], bundle["features"], bundle["labels"], artifact, classes, ), } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps(report, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()