Download src/train_probe.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/train_probe.py
- Command line
-
hf download hf://sneo/v802-classifier/src/train_probe.py
-
curl -L -o train_probe.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/train_probe.py
10.9 kB
| #!/usr/bin/env python3 | |
| """Select and fit a pure DINOv3 frozen-feature multilabel probe.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import json | |
| import platform | |
| import time | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedKFold | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.metrics import accuracy_score, average_precision_score, f1_score, hamming_loss, roc_auc_score | |
| from sklearn.preprocessing import StandardScaler | |
| def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray: | |
| grid = np.linspace(0.15, 0.85, 141) | |
| result = [] | |
| for column in range(y.shape[1]): | |
| scores = np.asarray([ | |
| f1_score(y[:, column], probabilities[:, column] >= threshold) | |
| for threshold in grid | |
| ]) | |
| best = np.flatnonzero(scores == scores.max()) | |
| result.append(grid[best[len(best) // 2]]) | |
| return np.asarray(result) | |
| def apply_thresholds(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray: | |
| predictions = (probabilities >= thresholds).astype(np.uint8) | |
| for index in range(len(predictions)): | |
| if predictions[index].sum() == 0: | |
| predictions[index, np.argmax(probabilities[index] / thresholds)] = 1 | |
| elif predictions[index].sum() > 3: | |
| positive = np.flatnonzero(predictions[index]) | |
| keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]] | |
| predictions[index] = 0 | |
| predictions[index, keep] = 1 | |
| return predictions | |
| def metrics(y: np.ndarray, probabilities: np.ndarray, thresholds: np.ndarray) -> dict[str, object]: | |
| predictions = apply_thresholds(probabilities, thresholds) | |
| return { | |
| "exact_match_accuracy": float(accuracy_score(y, predictions)), | |
| "micro_f1": float(f1_score(y, predictions, average="micro")), | |
| "macro_f1": float(f1_score(y, predictions, average="macro")), | |
| "per_class_f1": f1_score(y, predictions, average=None).tolist(), | |
| "mean_average_precision": float(average_precision_score(y, probabilities, average="macro")), | |
| "macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")), | |
| "hamming_loss": float(hamming_loss(y, predictions)), | |
| } | |
| def read_origins(path: Path) -> dict[str, str]: | |
| with path.open(newline="", encoding="utf-8-sig") as handle: | |
| return {row["file_name"]: row["origin"] for row in csv.DictReader(handle)} | |
| def fit_probe(x: np.ndarray, y: np.ndarray, c: float, scale: bool) -> dict[str, object]: | |
| scaler = StandardScaler().fit(x) if scale else None | |
| z = scaler.transform(x) if scaler is not None else x | |
| models = [] | |
| for column in range(y.shape[1]): | |
| model = LogisticRegression( | |
| C=c, | |
| solver="liblinear", | |
| dual=True, | |
| max_iter=3000, | |
| class_weight="balanced", | |
| random_state=42, | |
| ) | |
| model.fit(z, y[:, column]) | |
| models.append(model) | |
| return {"scaler": scaler, "models": models, "c": c, "scale": scale} | |
| def predict(probe: dict[str, object], x: np.ndarray) -> np.ndarray: | |
| scaler = probe["scaler"] | |
| z = scaler.transform(x) if scaler is not None else x | |
| return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]]) | |
| def file_sha256(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--development", type=Path, required=True) | |
| parser.add_argument("--model", type=Path, required=True) | |
| parser.add_argument("--output-dir", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=648208) | |
| args = parser.parse_args() | |
| started = time.time() | |
| bundle = np.load(args.features) | |
| feature_metadata_path = args.features.with_suffix(".json") | |
| feature_metadata = json.loads(feature_metadata_path.read_text(encoding="utf-8")) | |
| names = bundle["file_names"] | |
| x = bundle["features"].astype(np.float32) | |
| y = bundle["labels"].astype(np.uint8) | |
| classes = bundle["classes"] | |
| if feature_metadata.get("shape") != list(x.shape): | |
| raise ValueError("feature metadata shape mismatch") | |
| if not feature_metadata.get("model_sha256"): | |
| raise ValueError("feature metadata is missing the backbone SHA-256") | |
| embedded_model_digests = json.loads(str(bundle["model_file_sha256_json"])) | |
| if str(bundle["cache_key"]) != feature_metadata.get("cache_key"): | |
| raise ValueError("feature NPZ/metadata cache key mismatch") | |
| if str(bundle["feature_schema"]) != feature_metadata.get("feature_type"): | |
| raise ValueError("feature NPZ/metadata schema mismatch") | |
| if str(bundle["source_digest"]) != feature_metadata.get("source_digest"): | |
| raise ValueError("feature NPZ/metadata source digest mismatch") | |
| if embedded_model_digests != feature_metadata.get("model_file_sha256"): | |
| raise ValueError("feature NPZ/metadata model digest mismatch") | |
| name_to_index = {str(name): index for index, name in enumerate(names)} | |
| development = read_origins(args.development) | |
| development_idx = np.asarray([name_to_index[name] for name in development]) | |
| anchor_idx = np.asarray([ | |
| name_to_index[name] for name, origin in development.items() if origin == "old440" | |
| ]) | |
| target_idx = np.asarray([ | |
| name_to_index[name] for name, origin in development.items() if origin != "old440" | |
| ]) | |
| if len(development_idx) != len(development): | |
| raise ValueError("development manifest contains duplicate filenames") | |
| if len(anchor_idx) != 440 or len(target_idx) < 2: | |
| raise ValueError("expected 440 old440 anchors and at least two target-domain images") | |
| folds = list(MultilabelStratifiedKFold( | |
| n_splits=5, shuffle=True, random_state=args.seed | |
| ).split(np.zeros(len(target_idx)), y[target_idx])) | |
| candidates = [ | |
| *((True, c) for c in (0.001, 0.003, 0.01, 0.03, 0.1, 0.3)), | |
| *((False, c) for c in (0.03, 0.1, 0.3, 1.0, 3.0)), | |
| ] | |
| results = [] | |
| oof_by_name: dict[str, np.ndarray] = {} | |
| for number, (scale, c) in enumerate(candidates, start=1): | |
| probabilities = np.zeros_like(y[target_idx], dtype=np.float64) | |
| for train_local, valid_local in folds: | |
| train_idx = np.concatenate([anchor_idx, target_idx[train_local]]) | |
| valid_idx = target_idx[valid_local] | |
| probe = fit_probe(x[train_idx], y[train_idx], c, scale) | |
| probabilities[valid_local] = predict(probe, x[valid_idx]) | |
| name = f"dinov3-{'std' if scale else 'raw'}-C{c:g}" | |
| selection_thresholds = np.full(y.shape[1], 0.5) | |
| calibration_thresholds = choose_thresholds(y[target_idx], probabilities) | |
| result = { | |
| "name": name, | |
| "spec": {"c": c, "scale": scale}, | |
| "selection_thresholds": selection_thresholds.tolist(), | |
| "calibration_thresholds": calibration_thresholds.tolist(), | |
| } | |
| result.update(metrics(y[target_idx], probabilities, selection_thresholds)) | |
| result["calibrated_oof_metrics"] = metrics( | |
| y[target_idx], probabilities, calibration_thresholds | |
| ) | |
| results.append(result) | |
| oof_by_name[name] = probabilities | |
| print( | |
| f"[{number}/{len(candidates)}] {name} macro={result['macro_f1']:.4f} " | |
| f"exact={result['exact_match_accuracy']:.4f}", flush=True | |
| ) | |
| selected = max( | |
| results, | |
| key=lambda item: ( | |
| item["macro_f1"], item["mean_average_precision"], item["exact_match_accuracy"] | |
| ), | |
| ) | |
| thresholds = np.asarray(selected["calibration_thresholds"]) | |
| final_probe = fit_probe( | |
| x[development_idx], y[development_idx], | |
| float(selected["spec"]["c"]), bool(selected["spec"]["scale"]), | |
| ) | |
| artifact = { | |
| "format_version": 1, | |
| "method": "pure DINOv3 frozen four-view linear probe", | |
| "classes": classes, | |
| "thresholds": thresholds, | |
| "scaler": final_probe["scaler"], | |
| "models": final_probe["models"], | |
| "selected_name": selected["name"], | |
| "selected_spec": selected["spec"], | |
| "feature_model_path": str(args.model.resolve()), | |
| "feature_model_sha256": feature_metadata["model_sha256"], | |
| "feature_model_file_sha256": feature_metadata["model_file_sha256"], | |
| "feature_model_commit": feature_metadata.get("model_commit"), | |
| "feature_type": feature_metadata.get("feature_type"), | |
| "feature_dimension": int(x.shape[1]), | |
| "feature_file_sha256": file_sha256(args.features), | |
| "min_labels": 1, | |
| "max_labels": 3, | |
| } | |
| args.output_dir.mkdir(parents=True, exist_ok=True) | |
| joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3) | |
| selected_probabilities = oof_by_name[selected["name"]] | |
| np.savez_compressed( | |
| args.output_dir / "selected_oof_predictions.npz", | |
| file_names=names[target_idx], | |
| labels=y[target_idx], | |
| probabilities=selected_probabilities, | |
| predictions=apply_thresholds(selected_probabilities, thresholds), | |
| classes=classes, | |
| ) | |
| ordered = sorted( | |
| results, key=lambda item: (item["macro_f1"], item["mean_average_precision"]), reverse=True | |
| ) | |
| report = { | |
| "protocol": ( | |
| f"5-fold target-domain CV on {len(target_idx)} images; all 440 old images remain in each " | |
| "training fold; candidates are selected at fixed 0.5 thresholds; per-class " | |
| "thresholds are calibrated only after candidate selection; all manifests outside " | |
| "the supplied development set are excluded" | |
| ), | |
| "samples": { | |
| "old_always_train": len(anchor_idx), | |
| "target_cross_validated": len(target_idx), | |
| "development": len(development_idx), | |
| }, | |
| "selected": selected, | |
| "all_candidates": ordered, | |
| "classes": classes.tolist(), | |
| "seed": args.seed, | |
| "seconds": round(time.time() - started, 3), | |
| "python": platform.python_version(), | |
| "development_note": ( | |
| "calibrated_oof_metrics reuse OOF labels for threshold calibration and are not " | |
| "an unbiased final estimate; use only a separately locked holdout for final reporting" | |
| ), | |
| } | |
| (args.output_dir / "development_metrics.json").write_text( | |
| json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| print(json.dumps({"selected": selected, "seconds": report["seconds"]}, ensure_ascii=False, indent=2)) | |
| if __name__ == "__main__": | |
| main() | |