#!/usr/bin/env python3 """Select and fit a pure DINOv3 frozen-feature multilabel probe.""" from __future__ import annotations import argparse import csv import hashlib import json import platform import time from pathlib import Path import joblib import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score, average_precision_score, f1_score, hamming_loss, roc_auc_score from sklearn.preprocessing import StandardScaler def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray: grid = np.linspace(0.15, 0.85, 141) result = [] for column in range(y.shape[1]): scores = np.asarray([ f1_score(y[:, column], probabilities[:, column] >= threshold) for threshold in grid ]) best = np.flatnonzero(scores == scores.max()) result.append(grid[best[len(best) // 2]]) return np.asarray(result) def apply_thresholds(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray: predictions = (probabilities >= thresholds).astype(np.uint8) for index in range(len(predictions)): if predictions[index].sum() == 0: predictions[index, np.argmax(probabilities[index] / thresholds)] = 1 elif predictions[index].sum() > 3: positive = np.flatnonzero(predictions[index]) keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]] predictions[index] = 0 predictions[index, keep] = 1 return predictions def metrics(y: np.ndarray, probabilities: np.ndarray, thresholds: np.ndarray) -> dict[str, object]: predictions = apply_thresholds(probabilities, thresholds) return { "exact_match_accuracy": float(accuracy_score(y, predictions)), "micro_f1": float(f1_score(y, predictions, average="micro")), "macro_f1": float(f1_score(y, predictions, average="macro")), "per_class_f1": f1_score(y, predictions, average=None).tolist(), "mean_average_precision": float(average_precision_score(y, probabilities, average="macro")), "macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")), "hamming_loss": float(hamming_loss(y, predictions)), } def read_origins(path: Path) -> dict[str, str]: with path.open(newline="", encoding="utf-8-sig") as handle: return {row["file_name"]: row["origin"] for row in csv.DictReader(handle)} def fit_probe(x: np.ndarray, y: np.ndarray, c: float, scale: bool) -> dict[str, object]: scaler = StandardScaler().fit(x) if scale else None z = scaler.transform(x) if scaler is not None else x models = [] for column in range(y.shape[1]): model = LogisticRegression( C=c, solver="liblinear", dual=True, max_iter=3000, class_weight="balanced", random_state=42, ) model.fit(z, y[:, column]) models.append(model) return {"scaler": scaler, "models": models, "c": c, "scale": scale} def predict(probe: dict[str, object], x: np.ndarray) -> np.ndarray: scaler = probe["scaler"] z = scaler.transform(x) if scaler is not None else x return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]]) def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--development", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--seed", type=int, default=648208) args = parser.parse_args() started = time.time() bundle = np.load(args.features) feature_metadata_path = args.features.with_suffix(".json") feature_metadata = json.loads(feature_metadata_path.read_text(encoding="utf-8")) names = bundle["file_names"] x = bundle["features"].astype(np.float32) y = bundle["labels"].astype(np.uint8) classes = bundle["classes"] if feature_metadata.get("shape") != list(x.shape): raise ValueError("feature metadata shape mismatch") if not feature_metadata.get("model_sha256"): raise ValueError("feature metadata is missing the backbone SHA-256") embedded_model_digests = json.loads(str(bundle["model_file_sha256_json"])) if str(bundle["cache_key"]) != feature_metadata.get("cache_key"): raise ValueError("feature NPZ/metadata cache key mismatch") if str(bundle["feature_schema"]) != feature_metadata.get("feature_type"): raise ValueError("feature NPZ/metadata schema mismatch") if str(bundle["source_digest"]) != feature_metadata.get("source_digest"): raise ValueError("feature NPZ/metadata source digest mismatch") if embedded_model_digests != feature_metadata.get("model_file_sha256"): raise ValueError("feature NPZ/metadata model digest mismatch") name_to_index = {str(name): index for index, name in enumerate(names)} development = read_origins(args.development) development_idx = np.asarray([name_to_index[name] for name in development]) anchor_idx = np.asarray([ name_to_index[name] for name, origin in development.items() if origin == "old440" ]) target_idx = np.asarray([ name_to_index[name] for name, origin in development.items() if origin != "old440" ]) if len(development_idx) != len(development): raise ValueError("development manifest contains duplicate filenames") if len(anchor_idx) != 440 or len(target_idx) < 2: raise ValueError("expected 440 old440 anchors and at least two target-domain images") folds = list(MultilabelStratifiedKFold( n_splits=5, shuffle=True, random_state=args.seed ).split(np.zeros(len(target_idx)), y[target_idx])) candidates = [ *((True, c) for c in (0.001, 0.003, 0.01, 0.03, 0.1, 0.3)), *((False, c) for c in (0.03, 0.1, 0.3, 1.0, 3.0)), ] results = [] oof_by_name: dict[str, np.ndarray] = {} for number, (scale, c) in enumerate(candidates, start=1): probabilities = np.zeros_like(y[target_idx], dtype=np.float64) for train_local, valid_local in folds: train_idx = np.concatenate([anchor_idx, target_idx[train_local]]) valid_idx = target_idx[valid_local] probe = fit_probe(x[train_idx], y[train_idx], c, scale) probabilities[valid_local] = predict(probe, x[valid_idx]) name = f"dinov3-{'std' if scale else 'raw'}-C{c:g}" selection_thresholds = np.full(y.shape[1], 0.5) calibration_thresholds = choose_thresholds(y[target_idx], probabilities) result = { "name": name, "spec": {"c": c, "scale": scale}, "selection_thresholds": selection_thresholds.tolist(), "calibration_thresholds": calibration_thresholds.tolist(), } result.update(metrics(y[target_idx], probabilities, selection_thresholds)) result["calibrated_oof_metrics"] = metrics( y[target_idx], probabilities, calibration_thresholds ) results.append(result) oof_by_name[name] = probabilities print( f"[{number}/{len(candidates)}] {name} macro={result['macro_f1']:.4f} " f"exact={result['exact_match_accuracy']:.4f}", flush=True ) selected = max( results, key=lambda item: ( item["macro_f1"], item["mean_average_precision"], item["exact_match_accuracy"] ), ) thresholds = np.asarray(selected["calibration_thresholds"]) final_probe = fit_probe( x[development_idx], y[development_idx], float(selected["spec"]["c"]), bool(selected["spec"]["scale"]), ) artifact = { "format_version": 1, "method": "pure DINOv3 frozen four-view linear probe", "classes": classes, "thresholds": thresholds, "scaler": final_probe["scaler"], "models": final_probe["models"], "selected_name": selected["name"], "selected_spec": selected["spec"], "feature_model_path": str(args.model.resolve()), "feature_model_sha256": feature_metadata["model_sha256"], "feature_model_file_sha256": feature_metadata["model_file_sha256"], "feature_model_commit": feature_metadata.get("model_commit"), "feature_type": feature_metadata.get("feature_type"), "feature_dimension": int(x.shape[1]), "feature_file_sha256": file_sha256(args.features), "min_labels": 1, "max_labels": 3, } args.output_dir.mkdir(parents=True, exist_ok=True) joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3) selected_probabilities = oof_by_name[selected["name"]] np.savez_compressed( args.output_dir / "selected_oof_predictions.npz", file_names=names[target_idx], labels=y[target_idx], probabilities=selected_probabilities, predictions=apply_thresholds(selected_probabilities, thresholds), classes=classes, ) ordered = sorted( results, key=lambda item: (item["macro_f1"], item["mean_average_precision"]), reverse=True ) report = { "protocol": ( f"5-fold target-domain CV on {len(target_idx)} images; all 440 old images remain in each " "training fold; candidates are selected at fixed 0.5 thresholds; per-class " "thresholds are calibrated only after candidate selection; all manifests outside " "the supplied development set are excluded" ), "samples": { "old_always_train": len(anchor_idx), "target_cross_validated": len(target_idx), "development": len(development_idx), }, "selected": selected, "all_candidates": ordered, "classes": classes.tolist(), "seed": args.seed, "seconds": round(time.time() - started, 3), "python": platform.python_version(), "development_note": ( "calibrated_oof_metrics reuse OOF labels for threshold calibration and are not " "an unbiased final estimate; use only a separately locked holdout for final reporting" ), } (args.output_dir / "development_metrics.json").write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps({"selected": selected, "seconds": report["seconds"]}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()