Download src/evaluate.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 3.31 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/evaluate.py
- Command line
-
hf download hf://sneo/v802-classifier/src/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/evaluate.py
3.31 kB
| #!/usr/bin/env python3 | |
| """Evaluate a frozen DINOv3 probe on manifests excluded from model selection.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from train_probe import metrics, predict | |
| def file_sha256(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def evaluate_manifest( | |
| manifest: Path, names: np.ndarray, x: np.ndarray, y: np.ndarray, | |
| artifact: dict[str, object], classes: list[str], | |
| ) -> dict[str, object]: | |
| with manifest.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| name_to_index = {str(name): index for index, name in enumerate(names)} | |
| indices = np.asarray([name_to_index[row["file_name"]] for row in rows]) | |
| probabilities = predict(artifact, x[indices]) | |
| result = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"])) | |
| result["samples"] = len(indices) | |
| result["file_names"] = [row["file_name"] for row in rows] | |
| result["classes"] = classes | |
| return result | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--classifier", type=Path, required=True) | |
| parser.add_argument("--new-holdout", type=Path, required=True) | |
| parser.add_argument("--historical-holdout", type=Path, required=True) | |
| parser.add_argument("--output", type=Path, required=True) | |
| args = parser.parse_args() | |
| bundle = np.load(args.features) | |
| artifact = joblib.load(args.classifier) | |
| classes = [str(value) for value in bundle["classes"]] | |
| if classes != [str(value) for value in artifact["classes"]]: | |
| raise ValueError("feature/classifier class order mismatch") | |
| if int(bundle["features"].shape[1]) != int(artifact["feature_dimension"]): | |
| raise ValueError("feature/classifier dimension mismatch") | |
| if file_sha256(args.features) != artifact["feature_file_sha256"]: | |
| raise ValueError("feature file provenance mismatch") | |
| for model in artifact["models"]: | |
| if int(model.n_features_in_) != int(artifact["feature_dimension"]): | |
| raise ValueError("classifier coefficient dimension mismatch") | |
| report = { | |
| "protocol": ( | |
| "new_holdout20 is the primary untouched test; historical_holdout32 is reported " | |
| "only as a comparison with the already published v2 result" | |
| ), | |
| "selected_name": artifact["selected_name"], | |
| "new_holdout20": evaluate_manifest( | |
| args.new_holdout, bundle["file_names"], bundle["features"], bundle["labels"], | |
| artifact, classes, | |
| ), | |
| "historical_holdout32": evaluate_manifest( | |
| args.historical_holdout, bundle["file_names"], bundle["features"], bundle["labels"], | |
| artifact, classes, | |
| ), | |
| } | |
| args.output.parent.mkdir(parents=True, exist_ok=True) | |
| args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| print(json.dumps(report, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |