Download src/train_v801.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 28.8 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/train_v801.py
- Command line
-
hf download hf://sneo/v802-classifier/src/train_v801.py
-
curl -L -o train_v801.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/train_v801.py
28.8 kB
| #!/usr/bin/env python3 | |
| """Train an exact-match-aware structured probe on frozen DINOv3 features.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import hashlib | |
| import itertools | |
| import json | |
| import platform | |
| import time | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| from iterstrat.ml_stratifiers import MultilabelStratifiedKFold | |
| from sklearn.linear_model import LogisticRegression | |
| from sklearn.metrics import ( | |
| accuracy_score, | |
| average_precision_score, | |
| f1_score, | |
| hamming_loss, | |
| roc_auc_score, | |
| ) | |
| from sklearn.preprocessing import StandardScaler | |
| EPS = 1e-7 | |
| class Spec: | |
| variant: str | |
| c: float | |
| anchor_weight: float = 1.0 | |
| recent_weight: float = 1.0 | |
| class_weight: str | None = "balanced" | |
| scale: bool = True | |
| def name(self) -> str: | |
| balance = "bal" if self.class_weight else "plain" | |
| scaled = "std" if self.scale else "raw" | |
| return ( | |
| f"{self.variant}-{scaled}-C{self.c:g}-" | |
| f"old{self.anchor_weight:g}-recent{self.recent_weight:g}-{balance}" | |
| ) | |
| def sha256(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def l2_normalize(x: np.ndarray) -> np.ndarray: | |
| x = np.asarray(x, dtype=np.float32) | |
| return x / np.maximum(np.linalg.norm(x, axis=1, keepdims=True), 1e-12) | |
| def variant_from_descriptors( | |
| descriptors: np.ndarray, | |
| variant: str, | |
| legacy_features: np.ndarray | None = None, | |
| ) -> np.ndarray: | |
| descriptors = np.asarray(descriptors, dtype=np.float32) | |
| if descriptors.ndim != 4 or descriptors.shape[1:3] != (4, 4): | |
| raise ValueError(f"unexpected view descriptor shape: {descriptors.shape}") | |
| if variant == "legacy": | |
| if legacy_features is None: | |
| cls_patch = descriptors[:, :, :2].reshape(len(descriptors), 4, -1) | |
| legacy_features = np.concatenate( | |
| (cls_patch.mean(axis=1), cls_patch.max(axis=1)), axis=1 | |
| ) | |
| return l2_normalize(legacy_features) | |
| descriptor_count = { | |
| "meanmax_clspatchmax": 3, | |
| "meanmax_all": 4, | |
| "concat_clspatch": 2, | |
| "concat_all": 4, | |
| "full_clspatch": 2, | |
| "crops_meanmax_all": 4, | |
| }.get(variant) | |
| if descriptor_count is None: | |
| raise ValueError(f"unknown feature variant: {variant}") | |
| selected = descriptors[:, :, :descriptor_count] | |
| if variant.startswith("meanmax"): | |
| per_view = selected.reshape(len(selected), 4, -1) | |
| x = np.concatenate((per_view.mean(axis=1), per_view.max(axis=1)), axis=1) | |
| elif variant.startswith("concat"): | |
| x = selected.reshape(len(selected), -1) | |
| elif variant == "full_clspatch": | |
| x = selected[:, 0].reshape(len(selected), -1) | |
| else: | |
| crops = selected[:, 1:].reshape(len(selected), 3, -1) | |
| x = np.concatenate((crops.mean(axis=1), crops.max(axis=1)), axis=1) | |
| return l2_normalize(x) | |
| def make_variant(bundle: np.lib.npyio.NpzFile, variant: str) -> np.ndarray: | |
| return variant_from_descriptors( | |
| bundle["view_descriptors"], variant, bundle["features"] | |
| ) | |
| def fit_binary( | |
| x: np.ndarray, | |
| y: np.ndarray, | |
| spec: Spec, | |
| sample_weight: np.ndarray, | |
| ) -> dict[str, object]: | |
| scaler = StandardScaler().fit(x, sample_weight=sample_weight) if spec.scale else None | |
| z = scaler.transform(x) if scaler is not None else x | |
| models = [] | |
| for column in range(y.shape[1]): | |
| model = LogisticRegression( | |
| C=spec.c, | |
| solver="liblinear", | |
| dual=True, | |
| max_iter=3000, | |
| class_weight=spec.class_weight, | |
| random_state=801, | |
| ) | |
| model.fit(z, y[:, column], sample_weight=sample_weight) | |
| models.append(model) | |
| return {"scaler": scaler, "models": models} | |
| def binary_probabilities(probe: dict[str, object], x: np.ndarray) -> np.ndarray: | |
| scaler = probe["scaler"] | |
| z = scaler.transform(x) if scaler is not None else x | |
| return np.column_stack([model.predict_proba(z)[:, 1] for model in probe["models"]]) | |
| def artifact_probabilities( | |
| artifact: dict[str, object], descriptors: np.ndarray, legacy_features: np.ndarray | None = None | |
| ) -> np.ndarray: | |
| result = None | |
| for component in artifact["components"]: | |
| variant = str(component["spec"]["variant"]) | |
| x = variant_from_descriptors(descriptors, variant, legacy_features) | |
| if x.shape[1] != int(component["feature_dimension"]): | |
| raise ValueError(f"feature dimension mismatch for {variant}") | |
| probabilities = binary_probabilities(component, x) | |
| weighted = float(component["weight"]) * probabilities | |
| result = weighted if result is None else result + weighted | |
| local = artifact.get("local_correction") | |
| if local is not None: | |
| variant = str(local["variant"]) | |
| query = variant_from_descriptors(descriptors, variant, legacy_features) | |
| references = np.asarray(local["reference_features"], dtype=np.float32) | |
| reference_labels = np.asarray(local["reference_labels"], dtype=np.float64) | |
| if query.shape[1] != references.shape[1]: | |
| raise ValueError(f"local feature dimension mismatch for {variant}") | |
| k = int(local["k"]) | |
| temperature = float(local["temperature"]) | |
| similarity = query @ references.T | |
| neighbours = np.argpartition(similarity, -k, axis=1)[:, -k:] | |
| neighbour_similarity = np.take_along_axis(similarity, neighbours, axis=1) | |
| neighbour_similarity -= neighbour_similarity.max(axis=1, keepdims=True) | |
| weights = np.exp(neighbour_similarity / temperature) | |
| weights /= weights.sum(axis=1, keepdims=True) | |
| probabilities = np.einsum( | |
| "nk,nkc->nc", weights, reference_labels[neighbours] | |
| ) | |
| weighted = float(local["weight"]) * probabilities | |
| result = weighted if result is None else result + weighted | |
| if result is None: | |
| raise ValueError("classifier has no components") | |
| return result | |
| def structured_predictions( | |
| artifact: dict[str, object], probabilities: np.ndarray | |
| ) -> np.ndarray: | |
| thresholds = np.clip(np.asarray(artifact["thresholds"]), EPS, 1 - EPS) | |
| decoder = artifact["decoder"] | |
| q = float(decoder["threshold_bias_scale"]) | |
| logits = np.log( | |
| np.clip(probabilities, EPS, 1 - EPS) | |
| / np.clip(1 - probabilities, EPS, 1 - EPS) | |
| ) | |
| logits -= q * np.log(thresholds / (1 - thresholds)) | |
| labelsets = np.asarray(artifact["labelsets"], dtype=np.uint8) | |
| score = logits @ labelsets.T | |
| score += float(decoder["labelset_prior_weight"]) * np.asarray( | |
| artifact["labelset_log_prior"] | |
| ) | |
| cardinalities = labelsets.sum(axis=1).astype(int) - 1 | |
| card_probabilities = artifact["cardinality_model"].predict_proba( | |
| meta_features(probabilities) | |
| ) | |
| score += float(decoder["cardinality_weight"]) * np.log( | |
| np.clip(card_probabilities[:, cardinalities], EPS, 1) | |
| ) | |
| return labelsets[score.argmax(axis=1)] | |
| def choose_thresholds(y: np.ndarray, probabilities: np.ndarray) -> np.ndarray: | |
| grid = np.linspace(0.15, 0.85, 141) | |
| thresholds = [] | |
| for column in range(y.shape[1]): | |
| scores = np.asarray([ | |
| f1_score(y[:, column], probabilities[:, column] >= threshold) | |
| for threshold in grid | |
| ]) | |
| best = np.flatnonzero(scores == scores.max()) | |
| thresholds.append(grid[best[len(best) // 2]]) | |
| return np.asarray(thresholds) | |
| def threshold_predictions(probabilities: np.ndarray, thresholds: np.ndarray) -> np.ndarray: | |
| predictions = (probabilities >= thresholds).astype(np.uint8) | |
| for index, row in enumerate(predictions): | |
| if row.sum() == 0: | |
| predictions[index, np.argmax(probabilities[index] / thresholds)] = 1 | |
| elif row.sum() > 3: | |
| positive = np.flatnonzero(row) | |
| keep = positive[np.argpartition(probabilities[index, positive], -3)[-3:]] | |
| predictions[index] = 0 | |
| predictions[index, keep] = 1 | |
| return predictions | |
| def report_metrics( | |
| y: np.ndarray, probabilities: np.ndarray, predictions: np.ndarray | |
| ) -> dict[str, object]: | |
| return { | |
| "exact_match_accuracy": float(accuracy_score(y, predictions)), | |
| "micro_f1": float(f1_score(y, predictions, average="micro")), | |
| "macro_f1": float(f1_score(y, predictions, average="macro")), | |
| "per_class_f1": f1_score(y, predictions, average=None).tolist(), | |
| "mean_average_precision": float(average_precision_score(y, probabilities, average="macro")), | |
| "macro_roc_auc": float(roc_auc_score(y, probabilities, average="macro")), | |
| "hamming_loss": float(hamming_loss(y, predictions)), | |
| } | |
| def meta_features(probabilities: np.ndarray) -> np.ndarray: | |
| p = np.clip(probabilities, EPS, 1 - EPS) | |
| entropy = -(p * np.log(p) + (1 - p) * np.log(1 - p)).sum(axis=1) | |
| return np.column_stack((p, np.sort(p, axis=1), p.sum(axis=1), p.max(axis=1), entropy)) | |
| def all_labelsets(labels: int) -> np.ndarray: | |
| combinations = [ | |
| values | |
| for cardinality in (1, 2, 3) | |
| for values in itertools.combinations(range(labels), cardinality) | |
| ] | |
| result = np.zeros((len(combinations), labels), dtype=np.uint8) | |
| for index, values in enumerate(combinations): | |
| result[index, list(values)] = 1 | |
| return result | |
| def labelset_ids(y: np.ndarray, labelsets: np.ndarray) -> np.ndarray: | |
| lookup = {tuple(row): index for index, row in enumerate(labelsets)} | |
| try: | |
| return np.asarray([lookup[tuple(row)] for row in y], dtype=np.int64) | |
| except KeyError as error: | |
| raise ValueError(f"unsupported labelset: {error.args[0]}") from error | |
| def oof_probabilities( | |
| x: np.ndarray, | |
| y: np.ndarray, | |
| origins: np.ndarray, | |
| anchor_idx: np.ndarray, | |
| target_idx: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| spec: Spec, | |
| ) -> np.ndarray: | |
| result = np.zeros((len(target_idx), y.shape[1]), dtype=np.float64) | |
| for train_local, valid_local in folds: | |
| train_idx = np.concatenate((anchor_idx, target_idx[train_local])) | |
| weights = np.ones(len(train_idx), dtype=np.float64) | |
| weights[origins[train_idx] == "old440"] = spec.anchor_weight | |
| weights[origins[train_idx] == "incremental100"] = spec.recent_weight | |
| probe = fit_binary(x[train_idx], y[train_idx], spec, weights) | |
| result[valid_local] = binary_probabilities(probe, x[target_idx[valid_local]]) | |
| return result | |
| def crossfit_threshold_predictions( | |
| y: np.ndarray, | |
| probabilities: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| ) -> np.ndarray: | |
| predictions = np.zeros_like(y) | |
| for train, valid in folds: | |
| predictions[valid] = threshold_predictions( | |
| probabilities[valid], choose_thresholds(y[train], probabilities[train]) | |
| ) | |
| return predictions | |
| def structured_crossfit( | |
| y: np.ndarray, | |
| probabilities: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| q: float, | |
| prior_weight: float, | |
| cardinality_weight: float, | |
| smoothing: float = 1.0, | |
| ) -> np.ndarray: | |
| labelsets = all_labelsets(y.shape[1]) | |
| set_ids = labelset_ids(y, labelsets) | |
| cardinalities = labelsets.sum(axis=1).astype(int) - 1 | |
| predictions = np.zeros_like(y) | |
| features = meta_features(probabilities) | |
| for train, valid in folds: | |
| thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS) | |
| threshold_bias = q * np.log(thresholds / (1 - thresholds)) | |
| p = np.clip(probabilities[valid], EPS, 1 - EPS) | |
| logits = np.log(p / (1 - p)) - threshold_bias | |
| counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) | |
| log_prior = np.log((counts + smoothing) / (len(train) + smoothing * len(labelsets))) | |
| cardinality = LogisticRegression( | |
| C=1.0, max_iter=2000, class_weight="balanced", random_state=801 | |
| ).fit(features[train], y[train].sum(axis=1)) | |
| card_log_probability = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1)) | |
| score = logits @ labelsets.T | |
| score += prior_weight * log_prior | |
| score += cardinality_weight * card_log_probability[:, cardinalities] | |
| predictions[valid] = labelsets[score.argmax(axis=1)] | |
| return predictions | |
| def tune_structured_decoder( | |
| y: np.ndarray, | |
| probabilities: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| origins: np.ndarray, | |
| ) -> tuple[dict[str, float], np.ndarray, dict[str, object]]: | |
| labelsets = all_labelsets(y.shape[1]) | |
| set_ids = labelset_ids(y, labelsets) | |
| cardinalities = labelsets.sum(axis=1).astype(int) - 1 | |
| features = meta_features(probabilities) | |
| contexts = [] | |
| for train, valid in folds: | |
| thresholds = np.clip(choose_thresholds(y[train], probabilities[train]), EPS, 1 - EPS) | |
| threshold_bias = np.log(thresholds / (1 - thresholds)) | |
| p = np.clip(probabilities[valid], EPS, 1 - EPS) | |
| logits = np.log(p / (1 - p)) | |
| counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) | |
| log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets))) | |
| cardinality = LogisticRegression( | |
| C=1.0, max_iter=2000, class_weight="balanced", random_state=801 | |
| ).fit(features[train], y[train].sum(axis=1)) | |
| card_log_probability = np.log(np.clip( | |
| cardinality.predict_proba(features[valid]), EPS, 1 | |
| )) | |
| contexts.append((valid, logits, threshold_bias, log_prior, card_log_probability)) | |
| best: tuple[tuple[float, ...], dict[str, float], np.ndarray, dict[str, object]] | None = None | |
| recent = origins == "incremental100" | |
| for q in (0.0, 0.25, 0.5, 0.75, 1.0, 1.25): | |
| for prior_weight in (0.0, 0.1, 0.25, 0.5, 0.75, 1.0): | |
| for cardinality_weight in (0.0, 0.25, 0.5, 1.0, 1.5, 2.0): | |
| predictions = np.zeros_like(y) | |
| for valid, logits, threshold_bias, log_prior, card_log_probability in contexts: | |
| score = (logits - q * threshold_bias) @ labelsets.T | |
| score += prior_weight * log_prior | |
| score += cardinality_weight * card_log_probability[:, cardinalities] | |
| predictions[valid] = labelsets[score.argmax(axis=1)] | |
| metrics = report_metrics(y, probabilities, predictions) | |
| recent_exact = float(accuracy_score(y[recent], predictions[recent])) | |
| key = ( | |
| float(metrics["exact_match_accuracy"]), | |
| recent_exact, | |
| float(metrics["macro_f1"]), | |
| float(metrics["mean_average_precision"]), | |
| -prior_weight, | |
| -cardinality_weight, | |
| ) | |
| params = { | |
| "threshold_bias_scale": q, | |
| "labelset_prior_weight": prior_weight, | |
| "cardinality_weight": cardinality_weight, | |
| "prior_smoothing": 1.0, | |
| } | |
| if best is None or key > best[0]: | |
| metrics["incremental100_exact_match_accuracy"] = recent_exact | |
| best = key, params, predictions, metrics | |
| assert best is not None | |
| return best[1], best[2], best[3] | |
| def spec_dict(spec: Spec) -> dict[str, object]: | |
| return { | |
| "name": spec.name, | |
| "variant": spec.variant, | |
| "c": spec.c, | |
| "anchor_weight": spec.anchor_weight, | |
| "recent_weight": spec.recent_weight, | |
| "class_weight": spec.class_weight, | |
| "scale": spec.scale, | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--features", type=Path, required=True) | |
| parser.add_argument("--development", type=Path, required=True) | |
| parser.add_argument("--model", type=Path, required=True) | |
| parser.add_argument("--output-dir", type=Path, required=True) | |
| parser.add_argument("--seed", type=int, default=801208) | |
| args = parser.parse_args() | |
| started = time.time() | |
| bundle = np.load(args.features) | |
| metadata = json.loads(args.features.with_suffix(".json").read_text(encoding="utf-8")) | |
| names = bundle["file_names"].astype(str) | |
| y = bundle["labels"].astype(np.uint8) | |
| classes = bundle["classes"] | |
| name_to_index = {name: index for index, name in enumerate(names)} | |
| with args.development.open(newline="", encoding="utf-8-sig") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| development_idx = np.asarray([name_to_index[row["file_name"]] for row in rows]) | |
| origins = np.full(len(names), "excluded", dtype="U32") | |
| for row in rows: | |
| origins[name_to_index[row["file_name"]]] = row["origin"] | |
| anchor_idx = np.asarray([index for index in development_idx if origins[index] == "old440"]) | |
| target_idx = np.asarray([index for index in development_idx if origins[index] != "old440"]) | |
| if (len(development_idx), len(anchor_idx), len(target_idx)) != (736, 440, 296): | |
| raise ValueError("expected development/anchor/target sizes 736/440/296") | |
| folds = list(MultilabelStratifiedKFold( | |
| n_splits=5, shuffle=True, random_state=args.seed | |
| ).split(np.zeros(len(target_idx)), y[target_idx])) | |
| variants = ( | |
| "legacy", | |
| "meanmax_clspatchmax", | |
| "meanmax_all", | |
| "concat_clspatch", | |
| "concat_all", | |
| "full_clspatch", | |
| "crops_meanmax_all", | |
| ) | |
| features_by_variant = {variant: make_variant(bundle, variant) for variant in variants} | |
| oof_cache: dict[Spec, np.ndarray] = {} | |
| simple_reports: list[dict[str, object]] = [] | |
| def evaluate_spec(spec: Spec) -> dict[str, object]: | |
| if spec not in oof_cache: | |
| oof_cache[spec] = oof_probabilities( | |
| features_by_variant[spec.variant], y, origins, | |
| anchor_idx, target_idx, folds, spec, | |
| ) | |
| probabilities = oof_cache[spec] | |
| predictions = crossfit_threshold_predictions(y[target_idx], probabilities, folds) | |
| metrics = report_metrics(y[target_idx], probabilities, predictions) | |
| recent = origins[target_idx] == "incremental100" | |
| metrics["incremental100_exact_match_accuracy"] = float( | |
| accuracy_score(y[target_idx][recent], predictions[recent]) | |
| ) | |
| report = {"spec": spec_dict(spec), "crossfit_threshold_metrics": metrics} | |
| print( | |
| f"{spec.name}: exact={metrics['exact_match_accuracy']:.4f} " | |
| f"recent={metrics['incremental100_exact_match_accuracy']:.4f} " | |
| f"macro={metrics['macro_f1']:.4f}", | |
| flush=True, | |
| ) | |
| return report | |
| # Stage 1: screen representation families without touching locked manifests. | |
| for variant in variants: | |
| for c in (0.003, 0.01, 0.03): | |
| spec = Spec(variant=variant, c=c) | |
| simple_reports.append(evaluate_spec(spec)) | |
| def simple_key(report: dict[str, object]) -> tuple[float, ...]: | |
| metrics = report["crossfit_threshold_metrics"] | |
| return ( | |
| float(metrics["exact_match_accuracy"]), | |
| float(metrics["incremental100_exact_match_accuracy"]), | |
| float(metrics["macro_f1"]), | |
| float(metrics["mean_average_precision"]), | |
| ) | |
| best_by_variant: dict[str, dict[str, object]] = {} | |
| for report in simple_reports: | |
| variant = str(report["spec"]["variant"]) | |
| if variant not in best_by_variant or simple_key(report) > simple_key(best_by_variant[variant]): | |
| best_by_variant[variant] = report | |
| top_variants = [ | |
| str(report["spec"]["variant"]) | |
| for report in sorted(best_by_variant.values(), key=simple_key, reverse=True)[:2] | |
| ] | |
| print(f"Refining variants: {top_variants}", flush=True) | |
| # Stage 2: tune regularization, old-domain influence, and class balancing. | |
| for variant in top_variants: | |
| for c in (0.003, 0.01, 0.03, 0.1): | |
| for anchor_weight in (0.25, 0.5, 1.0): | |
| for class_weight in ("balanced", None): | |
| spec = Spec( | |
| variant=variant, | |
| c=c, | |
| anchor_weight=anchor_weight, | |
| class_weight=class_weight, | |
| ) | |
| if spec not in oof_cache: | |
| simple_reports.append(evaluate_spec(spec)) | |
| # A small, targeted domain-adaptation sweep for the primary evaluation domain. | |
| for c in (0.01, 0.03): | |
| for anchor_weight in (0.5, 1.0): | |
| for class_weight in ("balanced", None): | |
| spec = Spec( | |
| variant=variant, | |
| c=c, | |
| anchor_weight=anchor_weight, | |
| recent_weight=2.0, | |
| class_weight=class_weight, | |
| ) | |
| if spec not in oof_cache: | |
| simple_reports.append(evaluate_spec(spec)) | |
| simple_reports.sort(key=simple_key, reverse=True) | |
| structured_reports: list[dict[str, object]] = [] | |
| # Keep decoder search bounded and include distinct probability models. | |
| seen: set[Spec] = set() | |
| finalists: list[Spec] = [] | |
| for report in simple_reports: | |
| spec_values = report["spec"] | |
| spec = Spec( | |
| variant=str(spec_values["variant"]), | |
| c=float(spec_values["c"]), | |
| anchor_weight=float(spec_values["anchor_weight"]), | |
| recent_weight=float(spec_values["recent_weight"]), | |
| class_weight=spec_values["class_weight"], | |
| scale=bool(spec_values["scale"]), | |
| ) | |
| if spec not in seen: | |
| finalists.append(spec) | |
| seen.add(spec) | |
| if len(finalists) == 8: | |
| break | |
| for spec in finalists: | |
| params, predictions, metrics = tune_structured_decoder( | |
| y[target_idx], oof_cache[spec], folds, origins[target_idx] | |
| ) | |
| structured_reports.append({ | |
| "components": [{"weight": 1.0, "spec": spec_dict(spec)}], | |
| "decoder": params, | |
| "metrics": metrics, | |
| "probabilities": oof_cache[spec], | |
| "predictions": predictions, | |
| "specs": (spec,), | |
| "weights": (1.0,), | |
| }) | |
| print( | |
| f"structured {spec.name}: exact={metrics['exact_match_accuracy']:.4f} " | |
| f"recent={metrics['incremental100_exact_match_accuracy']:.4f}", flush=True | |
| ) | |
| # Stage 3: probability ensembles reduce fold/representation variance. | |
| ensemble_pool = finalists[:4] | |
| for first, second in itertools.combinations(ensemble_pool, 2): | |
| for first_weight in (0.25, 0.5, 0.75): | |
| probabilities = ( | |
| first_weight * oof_cache[first] + (1 - first_weight) * oof_cache[second] | |
| ) | |
| params, predictions, metrics = tune_structured_decoder( | |
| y[target_idx], probabilities, folds, origins[target_idx] | |
| ) | |
| structured_reports.append({ | |
| "components": [ | |
| {"weight": first_weight, "spec": spec_dict(first)}, | |
| {"weight": 1 - first_weight, "spec": spec_dict(second)}, | |
| ], | |
| "decoder": params, | |
| "metrics": metrics, | |
| "probabilities": probabilities, | |
| "predictions": predictions, | |
| "specs": (first, second), | |
| "weights": (first_weight, 1 - first_weight), | |
| }) | |
| def structured_key(report: dict[str, object]) -> tuple[float, ...]: | |
| metrics = report["metrics"] | |
| return ( | |
| float(metrics["exact_match_accuracy"]), | |
| float(metrics["incremental100_exact_match_accuracy"]), | |
| float(metrics["macro_f1"]), | |
| float(metrics["mean_average_precision"]), | |
| -float(len(report["components"])), | |
| ) | |
| structured_reports.sort(key=structured_key, reverse=True) | |
| selected = structured_reports[0] | |
| selected_probabilities = selected["probabilities"] | |
| selected_predictions = selected["predictions"] | |
| thresholds = choose_thresholds(y[target_idx], selected_probabilities) | |
| labelsets = all_labelsets(y.shape[1]) | |
| target_set_ids = labelset_ids(y[target_idx], labelsets) | |
| smoothing = float(selected["decoder"]["prior_smoothing"]) | |
| counts = np.bincount(target_set_ids, minlength=len(labelsets)).astype(np.float64) | |
| labelset_log_prior = np.log( | |
| (counts + smoothing) / (len(target_idx) + smoothing * len(labelsets)) | |
| ) | |
| cardinality_model = LogisticRegression( | |
| C=1.0, max_iter=2000, class_weight="balanced", random_state=801 | |
| ).fit(meta_features(selected_probabilities), y[target_idx].sum(axis=1)) | |
| final_components = [] | |
| for spec, weight in zip(selected["specs"], selected["weights"]): | |
| weights = np.ones(len(development_idx), dtype=np.float64) | |
| weights[origins[development_idx] == "old440"] = spec.anchor_weight | |
| weights[origins[development_idx] == "incremental100"] = spec.recent_weight | |
| probe = fit_binary( | |
| features_by_variant[spec.variant][development_idx], | |
| y[development_idx], spec, weights, | |
| ) | |
| final_components.append({ | |
| "weight": weight, | |
| "spec": spec_dict(spec), | |
| "feature_dimension": int(features_by_variant[spec.variant].shape[1]), | |
| "scaler": probe["scaler"], | |
| "models": probe["models"], | |
| }) | |
| artifact = { | |
| "format_version": 2, | |
| "version": "v801", | |
| "method": "DINOv3 structured multi-view exact-match probe", | |
| "classes": classes, | |
| "components": final_components, | |
| "thresholds": thresholds, | |
| "labelsets": labelsets, | |
| "labelset_log_prior": labelset_log_prior, | |
| "cardinality_model": cardinality_model, | |
| "decoder": selected["decoder"], | |
| "feature_model_path": str(args.model.resolve()), | |
| "feature_model_sha256": metadata["model_sha256"], | |
| "feature_model_file_sha256": metadata["model_file_sha256"], | |
| "feature_type": metadata["feature_type"], | |
| "feature_file_sha256": sha256(args.features), | |
| "descriptor_names": metadata["descriptor_names"], | |
| "min_labels": 1, | |
| "max_labels": 3, | |
| } | |
| args.output_dir.mkdir(parents=True, exist_ok=True) | |
| joblib.dump(artifact, args.output_dir / "classifier.joblib", compress=3) | |
| np.savez_compressed( | |
| args.output_dir / "selected_oof_predictions.npz", | |
| file_names=names[target_idx], | |
| labels=y[target_idx], | |
| probabilities=selected_probabilities, | |
| predictions=selected_predictions, | |
| classes=classes, | |
| ) | |
| compact_structured = [] | |
| for report in structured_reports: | |
| compact_structured.append({ | |
| "components": report["components"], | |
| "decoder": report["decoder"], | |
| "metrics": report["metrics"], | |
| }) | |
| development_report = { | |
| "version": "v801", | |
| "protocol": ( | |
| "Five-fold multilabel-stratified target-domain OOF selection on 296 images; " | |
| "440 old-domain anchors remain train-only in every fold. Feature conditioning, " | |
| "old-domain sample weight, binary-head regularization, class balancing, labelset " | |
| "prior, cardinality model, and probability ensembling are selected without reading " | |
| "either locked evaluation manifest. Cross-fit threshold/cardinality predictions " | |
| "are development-selection estimates, not final unbiased test estimates." | |
| ), | |
| "samples": {"development": 736, "old_anchors": 440, "target_oof": 296}, | |
| "selected": { | |
| "components": selected["components"], | |
| "decoder": selected["decoder"], | |
| "metrics": selected["metrics"], | |
| "thresholds_fit_on_all_target_oof": thresholds.tolist(), | |
| }, | |
| "structured_candidates": compact_structured, | |
| "simple_candidates": simple_reports, | |
| "feature_dimensions": { | |
| variant: int(values.shape[1]) for variant, values in features_by_variant.items() | |
| }, | |
| "classes": classes.tolist(), | |
| "seed": args.seed, | |
| "seconds": round(time.time() - started, 3), | |
| "python": platform.python_version(), | |
| "feature_file_sha256": sha256(args.features), | |
| } | |
| (args.output_dir / "development_metrics.json").write_text( | |
| json.dumps(development_report, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| print(json.dumps(development_report["selected"], ensure_ascii=False, indent=2), flush=True) | |
| if __name__ == "__main__": | |
| main() | |