v802-classifier / src /evaluate.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
3.31 kB
#!/usr/bin/env python3
"""Evaluate a frozen DINOv3 probe on manifests excluded from model selection."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
from pathlib import Path
import joblib
import numpy as np
from train_probe import metrics, predict
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def evaluate_manifest(
manifest: Path, names: np.ndarray, x: np.ndarray, y: np.ndarray,
artifact: dict[str, object], classes: list[str],
) -> dict[str, object]:
with manifest.open(newline="", encoding="utf-8-sig") as handle:
rows = list(csv.DictReader(handle))
name_to_index = {str(name): index for index, name in enumerate(names)}
indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
probabilities = predict(artifact, x[indices])
result = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"]))
result["samples"] = len(indices)
result["file_names"] = [row["file_name"] for row in rows]
result["classes"] = classes
return result
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--classifier", type=Path, required=True)
parser.add_argument("--new-holdout", type=Path, required=True)
parser.add_argument("--historical-holdout", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
bundle = np.load(args.features)
artifact = joblib.load(args.classifier)
classes = [str(value) for value in bundle["classes"]]
if classes != [str(value) for value in artifact["classes"]]:
raise ValueError("feature/classifier class order mismatch")
if int(bundle["features"].shape[1]) != int(artifact["feature_dimension"]):
raise ValueError("feature/classifier dimension mismatch")
if file_sha256(args.features) != artifact["feature_file_sha256"]:
raise ValueError("feature file provenance mismatch")
for model in artifact["models"]:
if int(model.n_features_in_) != int(artifact["feature_dimension"]):
raise ValueError("classifier coefficient dimension mismatch")
report = {
"protocol": (
"new_holdout20 is the primary untouched test; historical_holdout32 is reported "
"only as a comparison with the already published v2 result"
),
"selected_name": artifact["selected_name"],
"new_holdout20": evaluate_manifest(
args.new_holdout, bundle["file_names"], bundle["features"], bundle["labels"],
artifact, classes,
),
"historical_holdout32": evaluate_manifest(
args.historical_holdout, bundle["file_names"], bundle["features"], bundle["labels"],
artifact, classes,
),
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()