"""Shared data, fold, decoder and baseline helpers for v802 experiments.""" from __future__ import annotations import csv import json from dataclasses import dataclass from pathlib import Path import numpy as np from iterstrat.ml_stratifiers import MultilabelStratifiedKFold from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score from experiment_v801_knn import knn_oof from train_v801 import ( EPS, Spec, all_labelsets, labelset_ids, make_variant, meta_features, oof_probabilities, report_metrics, ) ROOT = Path(__file__).resolve().parents[1] SPLITS = ROOT / "artifacts" / "splits" / "v800" LEGACY_FEATURES = ROOT / "artifacts" / "features" / "dinov3-vitl16-all800-v801.npz" DEV_SEEDS = (801208, 801209, 801210, 801211) CHECK_SEEDS = (802001, 802002, 802003) V801_DECODER = {"prior_weight": 0.5, "cardinality_weight": 0.25} def read_split(name: str) -> list[dict[str, str]]: with (SPLITS / name).open(newline="", encoding="utf-8-sig") as handle: return list(csv.DictReader(handle)) @dataclass class Data: names: np.ndarray y: np.ndarray classes: tuple[str, ...] origins: np.ndarray development_idx: np.ndarray anchor_idx: np.ndarray target_idx: np.ndarray new_idx: np.ndarray historical_idx: np.ndarray def load_data(names: list[str] | np.ndarray) -> Data: """Index splits against a feature file order (list of file names).""" names = np.asarray(names).astype(str) lookup = {name: index for index, name in enumerate(names)} bundle = np.load(LEGACY_FEATURES) legacy_lookup = {n: i for i, n in enumerate(bundle["file_names"].astype(str))} y_legacy = bundle["labels"].astype(np.uint8) y = np.stack([y_legacy[legacy_lookup[name]] for name in names]) rows = read_split("development736.csv") development_idx = np.asarray([lookup[row["file_name"]] for row in rows]) origins = np.full(len(names), "excluded", dtype="U32") for row in rows: origins[lookup[row["file_name"]]] = row["origin"] anchor_idx = development_idx[origins[development_idx] == "old440"] target_idx = development_idx[origins[development_idx] != "old440"] new_idx = np.asarray([lookup[r["file_name"]] for r in read_split("new_holdout32.csv")]) historical_idx = np.asarray( [lookup[r["file_name"]] for r in read_split("historical_holdout32.csv")] ) assert (len(development_idx), len(anchor_idx), len(target_idx)) == (736, 440, 296) return Data( names, y, tuple(bundle["classes"].astype(str)), origins, development_idx, anchor_idx, target_idx, new_idx, historical_idx, ) def make_folds(data: Data, seed: int) -> list[tuple[np.ndarray, np.ndarray]]: return list( MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=seed).split( np.zeros(len(data.target_idx)), data.y[data.target_idx] ) ) def v801_oof(data: Data, seed: int, cache_dir: Path) -> dict[str, np.ndarray]: """Re-run the v801 legacy probe + kNN components on one seed's folds.""" cache = cache_dir / f"v801_oof_seed{seed}.npz" if cache.exists(): stored = np.load(cache) assert (stored["names"] == data.names[data.target_idx]).all() return {"legacy": stored["legacy"], "knn": stored["knn"]} bundle = np.load(LEGACY_FEATURES) legacy_names = bundle["file_names"].astype(str) order = np.asarray([{n: i for i, n in enumerate(legacy_names)}[n] for n in data.names]) legacy_x = make_variant(bundle, "legacy")[order] patch_x = make_variant(bundle, "meanmax_clspatchmax")[order] folds = make_folds(data, seed) legacy = oof_probabilities( legacy_x, data.y, data.origins, data.anchor_idx, data.target_idx, folds, Spec("legacy", 0.003, class_weight=None), ) knn = knn_oof(patch_x, data.y, data.anchor_idx, data.target_idx, folds, k=11, temperature=0.1) cache_dir.mkdir(parents=True, exist_ok=True) np.savez(cache, names=data.names[data.target_idx], legacy=legacy, knn=knn) return {"legacy": legacy, "knn": knn} def logit(p: np.ndarray) -> np.ndarray: p = np.clip(p, EPS, 1 - EPS) return np.log(p / (1 - p)) def decode_crossfit( y: np.ndarray, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], prior_weight: float, cardinality_weight: float, card_features: np.ndarray | None = None, card_c: float = 1.0, ) -> np.ndarray: """v801 structured decoder (q=0); optional extra cardinality features.""" labelsets = all_labelsets(y.shape[1]) set_ids = labelset_ids(y, labelsets) cardinalities = labelsets.sum(axis=1).astype(int) - 1 features = meta_features(probabilities) if card_features is not None: features = np.column_stack((features, card_features)) predictions = np.zeros_like(y) logits = logit(probabilities) for train, valid in folds: counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64) log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets))) cardinality = LogisticRegression( C=card_c, max_iter=3000, class_weight="balanced", random_state=801 ).fit(features[train], y[train].sum(axis=1)) card_log = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1)) score = logits[valid] @ labelsets.T score += prior_weight * log_prior score += cardinality_weight * card_log[:, cardinalities] predictions[valid] = labelsets[score.argmax(axis=1)] return predictions def evaluate( data: Data, probabilities: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], prior_weight: float = 0.5, cardinality_weight: float = 0.25, card_features: np.ndarray | None = None, ) -> dict[str, float]: y = data.y[data.target_idx] predictions = decode_crossfit( y, probabilities, folds, prior_weight, cardinality_weight, card_features ) metrics = report_metrics(y, probabilities, predictions) recent = data.origins[data.target_idx] == "incremental100" return { "exact": float(metrics["exact_match_accuracy"]), "recent_exact": float(accuracy_score(y[recent], predictions[recent])), "micro_f1": float(metrics["micro_f1"]), "map": float(metrics["mean_average_precision"]), "card_acc": float((predictions.sum(1) == y.sum(1)).mean()), } def summarize(results: list[dict[str, float]]) -> dict[str, float]: exact = [r["exact"] for r in results] return { "mean_exact": round(float(np.mean(exact)), 4), "min_exact": round(float(np.min(exact)), 4), "mean_recent": round(float(np.mean([r["recent_exact"] for r in results])), 4), "mean_map": round(float(np.mean([r["map"] for r in results])), 4), "mean_card": round(float(np.mean([r["card_acc"] for r in results])), 4), } def dump(path: Path, payload: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")