#!/usr/bin/env python3 """Train an exact-match-aware structured probe on frozen DINOv3 features.""" from __future__ import annotations import argparse import csv import hashlib import itertools import json import platform import time from dataclasses import dataclass from pathlib import Path import joblib import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, average_precision_score, f1_score, hamming_loss, roc_auc_score, ) from sklearn.preprocessing import StandardScaler EPS = 1e-7 @dataclass(frozen=True) class Spec: variant: str c: float anchor_weight: float = 1.0 recent_weight: float = 1.0 class_weight: str | None = "balanced" scale: bool = True @property def name(self) -> str: balance = "bal" if self.class_weight else "plain" scaled = "std" if self.scale else "raw" return ( f"{self.variant}-{scaled}-C{self.c:g}-" f"old{self.anchor_weight:g}-recent{self.recent_weight:g}-{balance}" ) def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def l2_normalize(x: np.ndarray) -> np.ndarray: x = np.asarray(x, dtype=np.float32) return x / np.maximum(np.linalg.norm(x, axis=1, keepdims=True), 1e-12) def variant_from_descriptors( descriptors: np.ndarray, variant: str, legacy_features: np.ndarray | None = None, ) -> np.ndarray: descriptors = np.asarray(descriptors, dtype=np.float32) if descriptors.ndim != 4 or descriptors.shape[1:3] != (4, 4): raise ValueError(f"unexpected view descriptor shape: {descriptors.shape}") if variant == "legacy": if legacy_features is None: cls_patch = descriptors[:, :, :2].reshape(len(descriptors), 4, -1) legacy_features = np.concatenate( (cls_patch.mean(axis=1), cls_patch.max(axis=1)), axis=1 ) return l2_normalize(legacy_features) descriptor_count = { "meanmax_clspatchmax": 3, "meanmax_all": 4, "concat_clspatch": 2, "concat_all": 4, "full_clspatch": 2, "crops_meanmax_all": 4, }.get(variant) if descriptor_count is None: raise ValueError(f"unknown feature variant: {variant}") selected = descriptors[:, :, :descriptor_count] if variant.startswith("meanmax"): per_view = selected.reshape(len(selected), 4, -1) x = np.concatenate((per_view.mean(axis=1), per_view.max(axis=1)), axis=1) elif variant.startswith("concat"): x = selected.reshape(len(selected), -1) elif variant == "full_clspatch": x = selected[:, 0].reshape(len(selected), -1) else: crops = selected[:, 1:].reshape(len(selected), 3, -1) x = np.concatenate((crops.mean(axis=1), crops.max(axis=1)), axis=1) return l2_normalize(x) def make_variant(bundle: np.lib.npyio.NpzFile, variant: str) -> np.ndarray: return variant_from_descriptors( bundle["view_descriptors"], variant, bundle["features"] ) def fit_binary( x: np.ndarray, y: np.ndarray, spec: Spec, sample_weight: np.ndarray, ) -> dict[str, object]: scaler = StandardScaler().fit(x, sample_weight=sample_weight) if spec.scale else None z = scaler.transform(x) if scaler is not None else x models = [] for column in range(y.shape[1]): model = LogisticRegression( C=spec.c, solver="liblinear", dual=True, max_iter=3000, class_weight=spec.class_weight, random_state=801, ) model.fit(z, y[:, column], sample_weight=sample_weight) models.append(model) return {"scaler": scaler, "models": models} def binary_probabilities(probe: dict[str, object], x: np.ndarray) -> np.ndarray: scaler = probe["scaler"] z = scaler.transform(x) if scaler is not None else x return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]]) def artifact_probabilities( artifact: dict[str, object], descriptors: np.ndarray, legacy_features: np.ndarray | None = None ) -> np.ndarray: result = None for component in artifact["components"]: variant = str(component["spec"]["variant"]) x = variant_from_descriptors(descriptors, variant, legacy_features) if x.shape[1] != int(component["feature_dimension"]): raise ValueError(f"feature dimension mismatch for {variant}") probabilities = binary_probabilities(component, x) weighted = float(component["weight"]) * probabilities result = weighted if result is None else result + weighted local = artifact.get("local_correction") if local is not None: variant = str(local["variant"]) query = variant_from_descriptors(descriptors, variant, legacy_features) references = np.asarray(local["reference_features"], dtype=np.float32) reference_labels = np.asarray(local["reference_labels"], dtype=np.float64) if query.shape[1] != references.shape[1]: raise ValueError(f"local feature dimension mismatch for {variant}") k = int(local["k"]) temperature = float(local["temperature"]) similarity = query @ references.T neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:] neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1) neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True) weights = np.exp(neighbour_similarity / temperature) weights /= weights.sum(axis=1, keepdims=True) probabilities = np.einsum( "nk,nkc->nc", weights, reference_labels[neighbours] ) weighted = float(local["weight"]) * probabilities result = weighted if result is None else result + weighted if result is None: raise ValueError("classifier has no components") return result def structured_predictions( artifact: dict[str, object], probabilities: np.ndarray ) -> np.ndarray: thresholds = np.clip(np.asarray(artifact["thresholds"]), EPS, 1 - EPS) decoder = artifact["decoder"] q = float(decoder["threshold_bias_scale"]) logits = np.log( np.clip(probabilities, EPS, 1 - EPS) / np.clip(1 - probabilities, EPS, 1 - EPS) ) logits -= q * np.log(thresholds / (1 - thresholds)) labelsets = np.asarray(artifact["labelsets"], dtype=np.uint8) score = logits @ labelsets.T score += float(decoder["labelset_prior_weight"]) * np.asarray( artifact["labelset_log_prior"] ) cardinalities = labelsets.sum(axis=1).astype(int) - 1 card_probabilities = artifact["cardinality_model"].predict_proba( meta_features(probabilities) ) score += float(decoder["cardinality_weight"]) * np.log( np.clip(card_probabilities[:, cardinalities], EPS, 1) ) return labelsets[score.argmax(axis=1)] def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray: grid = np.linspace(0.15, 0.85, 141) thresholds = [] for column in range(y.shape[1]): scores = np.asarray([ f1_score(y[:, column], probabilities[:, column] >= threshold) for threshold in grid ]) best = np.flatnonzero(scores == scores.max()) thresholds.append(grid[best[len(best) // 2]]) return np.asarray(thresholds) def threshold_predictions(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray: predictions = (probabilities >= thresholds).astype(np.uint8) for index, row in enumerate(predictions): if row.sum() == 0: predictions[index, np.argmax(probabilities[index] / thresholds)] = 1 elif row.sum() > 3: positive = np.flatnonzero(row) keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]] predictions[index] = 0 predictions[index, keep] = 1 return predictions def report_metrics( y: np.ndarray, probabilities: np.ndarray, predictions: np.ndarray ) -> dict[str, object]: return { "exact_match_accuracy": float(accuracy_score(y, predictions)), "micro_f1": float(f1_score(y, predictions, average="micro")), "macro_f1": float(f1_score(y, predictions, average="macro")), "per_class_f1": f1_score(y, predictions, average=None).tolist(), "mean_average_precision": float(average_precision_score(y, probabilities, average="macro")), "macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")), "hamming_loss": float(hamming_loss(y, predictions)), } def meta_features(probabilities: np.ndarray) -> np.ndarray: p = np.clip(probabilities, EPS, 1 - EPS) entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p)).sum(axis=1) return np.column_stack((p, np.sort(p, axis=1), p.sum(axis=1), p.max(axis=1), entropy)) def all_labelsets(labels: int) -> np.ndarray: combinations = [ values for cardinality in (1, 2, 3) for values in itertools.combinations(range(labels), cardinality) ] result = np.zeros((len(combinations), labels), dtype=np.uint8) for index, values in enumerate(combinations): result[index, list(values)] = 1 return result def labelset_ids(y: np.ndarray, labelsets: np.ndarray) -> np.ndarray: lookup = {tuple(row): index for index, row in enumerate(labelsets)} try: return np.asarray([lookup[tuple(row)] for row in y], dtype=np.int64) except KeyError as error: raise ValueError(f"unsupported labelset: {error.args[0]}") from error def oof_probabilities( x: np.ndarray, y: np.ndarray, origins: np.ndarray, anchor_idx: np.ndarray, target_idx: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], spec: Spec, ) -> np.ndarray: result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64) for train_local, valid_local in folds: train_idx = np.concatenate((anchor_idx, target_idx[train_local])) weights = np.ones(len(train_idx), dtype=np.float64) weights[origins[train_idx] == "old440"] = spec.anchor_weight weights[origins[train_idx] == "incremental100"] = spec.recent_weight probe = fit_binary(x[train_idx], y[train_idx], spec, weights) result[valid_local] = binary_probabilities(probe, x[target_idx[valid_local]]) return result def crossfit_threshold_predictions( y: np.ndarray, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], ) -> np.ndarray: predictions = np.zeros_like(y) for train, valid in folds: predictions[valid] = threshold_predictions( probabilities[valid], choose_thresholds(y[train], probabilities[train]) ) return predictions def structured_crossfit( y: np.ndarray, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], q: float, prior_weight: float, cardinality_weight: float, smoothing: float = 1.0, ) -> np.ndarray: labelsets = all_labelsets(y.shape[1]) set_ids = labelset_ids(y, labelsets) cardinalities = labelsets.sum(axis=1).astype(int) - 1 predictions = np.zeros_like(y) features = meta_features(probabilities) for train, valid in folds: thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS) threshold_bias = q * np.log(thresholds / (1 - thresholds)) p = np.clip(probabilities[valid], EPS, 1 - EPS) logits = np.log(p / (1 - p)) - threshold_bias counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) log_prior = np.log((counts + smoothing) / (len(train) + smoothing * len(labelsets))) cardinality = LogisticRegression( C=1.0, max_iter=2000, class_weight="balanced", random_state=801 ).fit(features[train], y[train].sum(axis=1)) card_log_probability = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1)) score = logits @ labelsets.T score += prior_weight * log_prior score += cardinality_weight * card_log_probability[:, cardinalities] predictions[valid] = labelsets[score.argmax(axis=1)] return predictions def tune_structured_decoder( y: np.ndarray, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], origins: np.ndarray, ) -> tuple[dict[str, float], np.ndarray, dict[str, object]]: labelsets = all_labelsets(y.shape[1]) set_ids = labelset_ids(y, labelsets) cardinalities = labelsets.sum(axis=1).astype(int) - 1 features = meta_features(probabilities) contexts = [] for train, valid in folds: thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS) threshold_bias = np.log(thresholds / (1 - thresholds)) p = np.clip(probabilities[valid], EPS, 1 - EPS) logits = np.log(p / (1 - p)) counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets))) cardinality = LogisticRegression( C=1.0, max_iter=2000, class_weight="balanced", random_state=801 ).fit(features[train], y[train].sum(axis=1)) card_log_probability = np.log(np.clip( cardinality.predict_proba(features[valid]), EPS, 1 )) contexts.append((valid, logits, threshold_bias, log_prior, card_log_probability)) best: tuple[tuple[float, ...], dict[str, float], np.ndarray, dict[str, object]] | None = None recent = origins == "incremental100" for q in (0.0, 0.25, 0.5, 0.75, 1.0, 1.25): for prior_weight in (0.0, 0.1, 0.25, 0.5, 0.75, 1.0): for cardinality_weight in (0.0, 0.25, 0.5, 1.0, 1.5, 2.0): predictions = np.zeros_like(y) for valid, logits, threshold_bias, log_prior, card_log_probability in contexts: score = (logits - q * threshold_bias) @ labelsets.T score += prior_weight * log_prior score += cardinality_weight * card_log_probability[:, cardinalities] predictions[valid] = labelsets[score.argmax(axis=1)] metrics = report_metrics(y, probabilities, predictions) recent_exact = float(accuracy_score(y[recent], predictions[recent])) key = ( float(metrics["exact_match_accuracy"]), recent_exact, float(metrics["macro_f1"]), float(metrics["mean_average_precision"]), -prior_weight, -cardinality_weight, ) params = { "threshold_bias_scale": q, "labelset_prior_weight": prior_weight, "cardinality_weight": cardinality_weight, "prior_smoothing": 1.0, } if best is None or key > best[0]: metrics["incremental100_exact_match_accuracy"] = recent_exact best = key, params, predictions, metrics assert best is not None return best[1], best[2], best[3] def spec_dict(spec: Spec) -> dict[str, object]: return { "name": spec.name, "variant": spec.variant, "c": spec.c, "anchor_weight": spec.anchor_weight, "recent_weight": spec.recent_weight, "class_weight": spec.class_weight, "scale": spec.scale, } def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--features", type=Path, required=True) parser.add_argument("--development", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--seed", type=int, default=801208) args = parser.parse_args() started = time.time() bundle = np.load(args.features) metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8")) names = bundle["file_names"].astype(str) y = bundle["labels"].astype(np.uint8) classes = bundle["classes"] name_to_index = {name: index for index, name in enumerate(names)} with args.development.open(newline="", encoding="utf-8-sig") as handle: rows = list(csv.DictReader(handle)) development_idx = np.asarray([name_to_index[row["file_name"]] for row in rows]) origins = np.full(len(names), "excluded", dtype="U32") for row in rows: origins[name_to_index[row["file_name"]]] = row["origin"] anchor_idx = np.asarray([index for index in development_idx if origins[index] == "old440"]) target_idx = np.asarray([index for index in development_idx if origins[index] != "old440"]) if (len(development_idx), len(anchor_idx), len(target_idx)) != (736, 440, 296): raise ValueError("expected development/anchor/target sizes 736/440/296") folds = list(MultilabelStratifiedKFold( n_splits=5, shuffle=True, random_state=args.seed ).split(np.zeros(len(target_idx)), y[target_idx])) variants = ( "legacy", "meanmax_clspatchmax", "meanmax_all", "concat_clspatch", "concat_all", "full_clspatch", "crops_meanmax_all", ) features_by_variant = {variant: make_variant(bundle, variant) for variant in variants} oof_cache: dict[Spec, np.ndarray] = {} simple_reports: list[dict[str, object]] = [] def evaluate_spec(spec: Spec) -> dict[str, object]: if spec not in oof_cache: oof_cache[spec] = oof_probabilities( features_by_variant[spec.variant], y, origins, anchor_idx, target_idx, folds, spec, ) probabilities = oof_cache[spec] predictions = crossfit_threshold_predictions(y[target_idx], probabilities, folds) metrics = report_metrics(y[target_idx], probabilities, predictions) recent = origins[target_idx] == "incremental100" metrics["incremental100_exact_match_accuracy"] = float( accuracy_score(y[target_idx][recent], predictions[recent]) ) report = {"spec": spec_dict(spec), "crossfit_threshold_metrics": metrics} print( f"{spec.name}: exact={metrics['exact_match_accuracy']:.4f} " f"recent={metrics['incremental100_exact_match_accuracy']:.4f} " f"macro={metrics['macro_f1']:.4f}", flush=True, ) return report # Stage 1: screen representation families without touching locked manifests. for variant in variants: for c in (0.003, 0.01, 0.03): spec = Spec(variant=variant, c=c) simple_reports.append(evaluate_spec(spec)) def simple_key(report: dict[str, object]) -> tuple[float, ...]: metrics = report["crossfit_threshold_metrics"] return ( float(metrics["exact_match_accuracy"]), float(metrics["incremental100_exact_match_accuracy"]), float(metrics["macro_f1"]), float(metrics["mean_average_precision"]), ) best_by_variant: dict[str, dict[str, object]] = {} for report in simple_reports: variant = str(report["spec"]["variant"]) if variant not in best_by_variant or simple_key(report) > simple_key(best_by_variant[variant]): best_by_variant[variant] = report top_variants = [ str(report["spec"]["variant"]) for report in sorted(best_by_variant.values(), key=simple_key, reverse=True)[:2] ] print(f"Refining variants: {top_variants}", flush=True) # Stage 2: tune regularization, old-domain influence, and class balancing. for variant in top_variants: for c in (0.003, 0.01, 0.03, 0.1): for anchor_weight in (0.25, 0.5, 1.0): for class_weight in ("balanced", None): spec = Spec( variant=variant, c=c, anchor_weight=anchor_weight, class_weight=class_weight, ) if spec not in oof_cache: simple_reports.append(evaluate_spec(spec)) # A small, targeted domain-adaptation sweep for the primary evaluation domain. for c in (0.01, 0.03): for anchor_weight in (0.5, 1.0): for class_weight in ("balanced", None): spec = Spec( variant=variant, c=c, anchor_weight=anchor_weight, recent_weight=2.0, class_weight=class_weight, ) if spec not in oof_cache: simple_reports.append(evaluate_spec(spec)) simple_reports.sort(key=simple_key, reverse=True) structured_reports: list[dict[str, object]] = [] # Keep decoder search bounded and include distinct probability models. seen: set[Spec] = set() finalists: list[Spec] = [] for report in simple_reports: spec_values = report["spec"] spec = Spec( variant=str(spec_values["variant"]), c=float(spec_values["c"]), anchor_weight=float(spec_values["anchor_weight"]), recent_weight=float(spec_values["recent_weight"]), class_weight=spec_values["class_weight"], scale=bool(spec_values["scale"]), ) if spec not in seen: finalists.append(spec) seen.add(spec) if len(finalists) == 8: break for spec in finalists: params, predictions, metrics = tune_structured_decoder( y[target_idx], oof_cache[spec], folds, origins[target_idx] ) structured_reports.append({ "components": [{"weight": 1.0, "spec": spec_dict(spec)}], "decoder": params, "metrics": metrics, "probabilities": oof_cache[spec], "predictions": predictions, "specs": (spec,), "weights": (1.0,), }) print( f"structured {spec.name}: exact={metrics['exact_match_accuracy']:.4f} " f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True ) # Stage 3: probability ensembles reduce fold/representation variance. ensemble_pool = finalists[:4] for first, second in itertools.combinations(ensemble_pool, 2): for first_weight in (0.25, 0.5, 0.75): probabilities = ( first_weight * oof_cache[first] + (1 - first_weight) * oof_cache[second] ) params, predictions, metrics = tune_structured_decoder( y[target_idx], probabilities, folds, origins[target_idx] ) structured_reports.append({ "components": [ {"weight": first_weight, "spec": spec_dict(first)}, {"weight": 1 - first_weight, "spec": spec_dict(second)}, ], "decoder": params, "metrics": metrics, "probabilities": probabilities, "predictions": predictions, "specs": (first, second), "weights": (first_weight, 1 - first_weight), }) def structured_key(report: dict[str, object]) -> tuple[float, ...]: metrics = report["metrics"] return ( float(metrics["exact_match_accuracy"]), float(metrics["incremental100_exact_match_accuracy"]), float(metrics["macro_f1"]), float(metrics["mean_average_precision"]), -float(len(report["components"])), ) structured_reports.sort(key=structured_key, reverse=True) selected = structured_reports[0] selected_probabilities = selected["probabilities"] selected_predictions = selected["predictions"] thresholds = choose_thresholds(y[target_idx], selected_probabilities) labelsets = all_labelsets(y.shape[1]) target_set_ids = labelset_ids(y[target_idx], labelsets) smoothing = float(selected["decoder"]["prior_smoothing"]) counts = np.bincount(target_set_ids, minlength=len(labelsets)).astype(np.float64) labelset_log_prior = np.log( (counts + smoothing) / (len(target_idx) + smoothing * len(labelsets)) ) cardinality_model = LogisticRegression( C=1.0, max_iter=2000, class_weight="balanced", random_state=801 ).fit(meta_features(selected_probabilities), y[target_idx].sum(axis=1)) final_components = [] for spec, weight in zip(selected["specs"], selected["weights"]): weights = np.ones(len(development_idx), dtype=np.float64) weights[origins[development_idx] == "old440"] = spec.anchor_weight weights[origins[development_idx] == "incremental100"] = spec.recent_weight probe = fit_binary( features_by_variant[spec.variant][development_idx], y[development_idx], spec, weights, ) final_components.append({ "weight": weight, "spec": spec_dict(spec), "feature_dimension": int(features_by_variant[spec.variant].shape[1]), "scaler": probe["scaler"], "models": probe["models"], }) artifact = { "format_version": 2, "version": "v801", "method": "DINOv3 structured multi-view exact-match probe", "classes": classes, "components": final_components, "thresholds": thresholds, "labelsets": labelsets, "labelset_log_prior": labelset_log_prior, "cardinality_model": cardinality_model, "decoder": selected["decoder"], "feature_model_path": str(args.model.resolve()), "feature_model_sha256": metadata["model_sha256"], "feature_model_file_sha256": metadata["model_file_sha256"], "feature_type": metadata["feature_type"], "feature_file_sha256": sha256(args.features), "descriptor_names": metadata["descriptor_names"], "min_labels": 1, "max_labels": 3, } args.output_dir.mkdir(parents=True, exist_ok=True) joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3) np.savez_compressed( args.output_dir / "selected_oof_predictions.npz", file_names=names[target_idx], labels=y[target_idx], probabilities=selected_probabilities, predictions=selected_predictions, classes=classes, ) compact_structured = [] for report in structured_reports: compact_structured.append({ "components": report["components"], "decoder": report["decoder"], "metrics": report["metrics"], }) development_report = { "version": "v801", "protocol": ( "Five-fold multilabel-stratified target-domain OOF selection on 296 images; " "440 old-domain anchors remain train-only in every fold. Feature conditioning, " "old-domain sample weight, binary-head regularization, class balancing, labelset " "prior, cardinality model, and probability ensembling are selected without reading " "either locked evaluation manifest. Cross-fit threshold/cardinality predictions " "are development-selection estimates, not final unbiased test estimates." ), "samples": {"development": 736, "old_anchors": 440, "target_oof": 296}, "selected": { "components": selected["components"], "decoder": selected["decoder"], "metrics": selected["metrics"], "thresholds_fit_on_all_target_oof": thresholds.tolist(), }, "structured_candidates": compact_structured, "simple_candidates": simple_reports, "feature_dimensions": { variant: int(values.shape[1]) for variant, values in features_by_variant.items() }, "classes": classes.tolist(), "seed": args.seed, "seconds": round(time.time() - started, 3), "python": platform.python_version(), "feature_file_sha256": sha256(args.features), } (args.output_dir / "development_metrics.json").write_text( json.dumps(development_report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print(json.dumps(development_report["selected"], ensure_ascii=False, indent=2), flush=True) if __name__ == "__main__": main()