File size: 3,307 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/usr/bin/env python3
"""Evaluate a frozen DINOv3 probe on manifests excluded from model selection."""

from __future__ import annotations

import argparse
import csv
import hashlib
import json
from pathlib import Path

import joblib
import numpy as np

from train_probe import metrics, predict


def file_sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def evaluate_manifest(
    manifest: Path, names: np.ndarray, x: np.ndarray, y: np.ndarray,
    artifact: dict[str, object], classes: list[str],
) -> dict[str, object]:
    with manifest.open(newline="", encoding="utf-8-sig") as handle:
        rows = list(csv.DictReader(handle))
    name_to_index = {str(name): index for index, name in enumerate(names)}
    indices = np.asarray([name_to_index[row["file_name"]] for row in rows])
    probabilities = predict(artifact, x[indices])
    result = metrics(y[indices], probabilities, np.asarray(artifact["thresholds"]))
    result["samples"] = len(indices)
    result["file_names"] = [row["file_name"] for row in rows]
    result["classes"] = classes
    return result


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--features", type=Path, required=True)
    parser.add_argument("--classifier", type=Path, required=True)
    parser.add_argument("--new-holdout", type=Path, required=True)
    parser.add_argument("--historical-holdout", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    args = parser.parse_args()

    bundle = np.load(args.features)
    artifact = joblib.load(args.classifier)
    classes = [str(value) for value in bundle["classes"]]
    if classes != [str(value) for value in artifact["classes"]]:
        raise ValueError("feature/classifier class order mismatch")
    if int(bundle["features"].shape[1]) != int(artifact["feature_dimension"]):
        raise ValueError("feature/classifier dimension mismatch")
    if file_sha256(args.features) != artifact["feature_file_sha256"]:
        raise ValueError("feature file provenance mismatch")
    for model in artifact["models"]:
        if int(model.n_features_in_) != int(artifact["feature_dimension"]):
            raise ValueError("classifier coefficient dimension mismatch")
    report = {
        "protocol": (
            "new_holdout20 is the primary untouched test; historical_holdout32 is reported "
            "only as a comparison with the already published v2 result"
        ),
        "selected_name": artifact["selected_name"],
        "new_holdout20": evaluate_manifest(
            args.new_holdout, bundle["file_names"], bundle["features"], bundle["labels"],
            artifact, classes,
        ),
        "historical_holdout32": evaluate_manifest(
            args.historical_holdout, bundle["file_names"], bundle["features"], bundle["labels"],
            artifact, classes,
        ),
    }
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()