v802-classifier / src /train_probe.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
10.9 kB
#!/usr/bin/env python3
"""Select and fit a pure DINOv3 frozen-feature multilabel probe."""
from __future__ import annotations
import argparse
import csv
import hashlib
import json
import platform
import time
from pathlib import Path
import joblib
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, average_precision_score, f1_score, hamming_loss, roc_auc_score
from sklearn.preprocessing import StandardScaler
def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray:
grid = np.linspace(0.15, 0.85, 141)
result = []
for column in range(y.shape[1]):
scores = np.asarray([
f1_score(y[:, column], probabilities[:, column] >= threshold)
for threshold in grid
])
best = np.flatnonzero(scores == scores.max())
result.append(grid[best[len(best) // 2]])
return np.asarray(result)
def apply_thresholds(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray:
predictions = (probabilities >= thresholds).astype(np.uint8)
for index in range(len(predictions)):
if predictions[index].sum() == 0:
predictions[index, np.argmax(probabilities[index] / thresholds)] = 1
elif predictions[index].sum() > 3:
positive = np.flatnonzero(predictions[index])
keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]]
predictions[index] = 0
predictions[index, keep] = 1
return predictions
def metrics(y: np.ndarray, probabilities: np.ndarray, thresholds: np.ndarray) -> dict[str, object]:
predictions = apply_thresholds(probabilities, thresholds)
return {
"exact_match_accuracy": float(accuracy_score(y, predictions)),
"micro_f1": float(f1_score(y, predictions, average="micro")),
"macro_f1": float(f1_score(y, predictions, average="macro")),
"per_class_f1": f1_score(y, predictions, average=None).tolist(),
"mean_average_precision": float(average_precision_score(y, probabilities, average="macro")),
"macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")),
"hamming_loss": float(hamming_loss(y, predictions)),
}
def read_origins(path: Path) -> dict[str, str]:
with path.open(newline="", encoding="utf-8-sig") as handle:
return {row["file_name"]: row["origin"] for row in csv.DictReader(handle)}
def fit_probe(x: np.ndarray, y: np.ndarray, c: float, scale: bool) -> dict[str, object]:
scaler = StandardScaler().fit(x) if scale else None
z = scaler.transform(x) if scaler is not None else x
models = []
for column in range(y.shape[1]):
model = LogisticRegression(
C=c,
solver="liblinear",
dual=True,
max_iter=3000,
class_weight="balanced",
random_state=42,
)
model.fit(z, y[:, column])
models.append(model)
return {"scaler": scaler, "models": models, "c": c, "scale": scale}
def predict(probe: dict[str, object], x: np.ndarray) -> np.ndarray:
scaler = probe["scaler"]
z = scaler.transform(x) if scaler is not None else x
return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]])
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--features", type=Path, required=True)
parser.add_argument("--development", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--seed", type=int, default=648208)
args = parser.parse_args()
started = time.time()
bundle = np.load(args.features)
feature_metadata_path = args.features.with_suffix(".json")
feature_metadata = json.loads(feature_metadata_path.read_text(encoding="utf-8"))
names = bundle["file_names"]
x = bundle["features"].astype(np.float32)
y = bundle["labels"].astype(np.uint8)
classes = bundle["classes"]
if feature_metadata.get("shape") != list(x.shape):
raise ValueError("feature metadata shape mismatch")
if not feature_metadata.get("model_sha256"):
raise ValueError("feature metadata is missing the backbone SHA-256")
embedded_model_digests = json.loads(str(bundle["model_file_sha256_json"]))
if str(bundle["cache_key"]) != feature_metadata.get("cache_key"):
raise ValueError("feature NPZ/metadata cache key mismatch")
if str(bundle["feature_schema"]) != feature_metadata.get("feature_type"):
raise ValueError("feature NPZ/metadata schema mismatch")
if str(bundle["source_digest"]) != feature_metadata.get("source_digest"):
raise ValueError("feature NPZ/metadata source digest mismatch")
if embedded_model_digests != feature_metadata.get("model_file_sha256"):
raise ValueError("feature NPZ/metadata model digest mismatch")
name_to_index = {str(name): index for index, name in enumerate(names)}
development = read_origins(args.development)
development_idx = np.asarray([name_to_index[name] for name in development])
anchor_idx = np.asarray([
name_to_index[name] for name, origin in development.items() if origin == "old440"
])
target_idx = np.asarray([
name_to_index[name] for name, origin in development.items() if origin != "old440"
])
if len(development_idx) != len(development):
raise ValueError("development manifest contains duplicate filenames")
if len(anchor_idx) != 440 or len(target_idx) < 2:
raise ValueError("expected 440 old440 anchors and at least two target-domain images")
folds = list(MultilabelStratifiedKFold(
n_splits=5, shuffle=True, random_state=args.seed
).split(np.zeros(len(target_idx)), y[target_idx]))
candidates = [
*((True, c) for c in (0.001, 0.003, 0.01, 0.03, 0.1, 0.3)),
*((False, c) for c in (0.03, 0.1, 0.3, 1.0, 3.0)),
]
results = []
oof_by_name: dict[str, np.ndarray] = {}
for number, (scale, c) in enumerate(candidates, start=1):
probabilities = np.zeros_like(y[target_idx], dtype=np.float64)
for train_local, valid_local in folds:
train_idx = np.concatenate([anchor_idx, target_idx[train_local]])
valid_idx = target_idx[valid_local]
probe = fit_probe(x[train_idx], y[train_idx], c, scale)
probabilities[valid_local] = predict(probe, x[valid_idx])
name = f"dinov3-{'std' if scale else 'raw'}-C{c:g}"
selection_thresholds = np.full(y.shape[1], 0.5)
calibration_thresholds = choose_thresholds(y[target_idx], probabilities)
result = {
"name": name,
"spec": {"c": c, "scale": scale},
"selection_thresholds": selection_thresholds.tolist(),
"calibration_thresholds": calibration_thresholds.tolist(),
}
result.update(metrics(y[target_idx], probabilities, selection_thresholds))
result["calibrated_oof_metrics"] = metrics(
y[target_idx], probabilities, calibration_thresholds
)
results.append(result)
oof_by_name[name] = probabilities
print(
f"[{number}/{len(candidates)}] {name} macro={result['macro_f1']:.4f} "
f"exact={result['exact_match_accuracy']:.4f}", flush=True
)
selected = max(
results,
key=lambda item: (
item["macro_f1"], item["mean_average_precision"], item["exact_match_accuracy"]
),
)
thresholds = np.asarray(selected["calibration_thresholds"])
final_probe = fit_probe(
x[development_idx], y[development_idx],
float(selected["spec"]["c"]), bool(selected["spec"]["scale"]),
)
artifact = {
"format_version": 1,
"method": "pure DINOv3 frozen four-view linear probe",
"classes": classes,
"thresholds": thresholds,
"scaler": final_probe["scaler"],
"models": final_probe["models"],
"selected_name": selected["name"],
"selected_spec": selected["spec"],
"feature_model_path": str(args.model.resolve()),
"feature_model_sha256": feature_metadata["model_sha256"],
"feature_model_file_sha256": feature_metadata["model_file_sha256"],
"feature_model_commit": feature_metadata.get("model_commit"),
"feature_type": feature_metadata.get("feature_type"),
"feature_dimension": int(x.shape[1]),
"feature_file_sha256": file_sha256(args.features),
"min_labels": 1,
"max_labels": 3,
}
args.output_dir.mkdir(parents=True, exist_ok=True)
joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3)
selected_probabilities = oof_by_name[selected["name"]]
np.savez_compressed(
args.output_dir / "selected_oof_predictions.npz",
file_names=names[target_idx],
labels=y[target_idx],
probabilities=selected_probabilities,
predictions=apply_thresholds(selected_probabilities, thresholds),
classes=classes,
)
ordered = sorted(
results, key=lambda item: (item["macro_f1"], item["mean_average_precision"]), reverse=True
)
report = {
"protocol": (
f"5-fold target-domain CV on {len(target_idx)} images; all 440 old images remain in each "
"training fold; candidates are selected at fixed 0.5 thresholds; per-class "
"thresholds are calibrated only after candidate selection; all manifests outside "
"the supplied development set are excluded"
),
"samples": {
"old_always_train": len(anchor_idx),
"target_cross_validated": len(target_idx),
"development": len(development_idx),
},
"selected": selected,
"all_candidates": ordered,
"classes": classes.tolist(),
"seed": args.seed,
"seconds": round(time.time() - started, 3),
"python": platform.python_version(),
"development_note": (
"calibrated_oof_metrics reuse OOF labels for threshold calibration and are not "
"an unbiased final estimate; use only a separately locked holdout for final reporting"
),
}
(args.output_dir / "development_metrics.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps({"selected": selected, "seconds": report["seconds"]}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()