File size: 7,109 Bytes
59839e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | """Shared data, fold, decoder and baseline helpers for v802 experiments."""
from __future__ import annotations
import csv
import json
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from experiment_v801_knn import knn_oof
from train_v801 import (
EPS,
Spec,
all_labelsets,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
)
ROOT = Path(__file__).resolve().parents[1]
SPLITS = ROOT / "artifacts" / "splits" / "v800"
LEGACY_FEATURES = ROOT / "artifacts" / "features" / "dinov3-vitl16-all800-v801.npz"
DEV_SEEDS = (801208, 801209, 801210, 801211)
CHECK_SEEDS = (802001, 802002, 802003)
V801_DECODER = {"prior_weight": 0.5, "cardinality_weight": 0.25}
def read_split(name: str) -> list[dict[str, str]]:
with (SPLITS / name).open(newline="", encoding="utf-8-sig") as handle:
return list(csv.DictReader(handle))
@dataclass
class Data:
names: np.ndarray
y: np.ndarray
classes: tuple[str, ...]
origins: np.ndarray
development_idx: np.ndarray
anchor_idx: np.ndarray
target_idx: np.ndarray
new_idx: np.ndarray
historical_idx: np.ndarray
def load_data(names: list[str] | np.ndarray) -> Data:
"""Index splits against a feature file order (list of file names)."""
names = np.asarray(names).astype(str)
lookup = {name: index for index, name in enumerate(names)}
bundle = np.load(LEGACY_FEATURES)
legacy_lookup = {n: i for i, n in enumerate(bundle["file_names"].astype(str))}
y_legacy = bundle["labels"].astype(np.uint8)
y = np.stack([y_legacy[legacy_lookup[name]] for name in names])
rows = read_split("development736.csv")
development_idx = np.asarray([lookup[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
new_idx = np.asarray([lookup[r["file_name"]] for r in read_split("new_holdout32.csv")])
historical_idx = np.asarray(
[lookup[r["file_name"]] for r in read_split("historical_holdout32.csv")]
)
assert (len(development_idx), len(anchor_idx), len(target_idx)) == (736, 440, 296)
return Data(
names, y, tuple(bundle["classes"].astype(str)), origins, development_idx,
anchor_idx, target_idx, new_idx, historical_idx,
)
def make_folds(data: Data, seed: int) -> list[tuple[np.ndarray, np.ndarray]]:
return list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=seed).split(
np.zeros(len(data.target_idx)), data.y[data.target_idx]
)
)
def v801_oof(data: Data, seed: int, cache_dir: Path) -> dict[str, np.ndarray]:
"""Re-run the v801 legacy probe + kNN components on one seed's folds."""
cache = cache_dir / f"v801_oof_seed{seed}.npz"
if cache.exists():
stored = np.load(cache)
assert (stored["names"] == data.names[data.target_idx]).all()
return {"legacy": stored["legacy"], "knn": stored["knn"]}
bundle = np.load(LEGACY_FEATURES)
legacy_names = bundle["file_names"].astype(str)
order = np.asarray([{n: i for i, n in enumerate(legacy_names)}[n] for n in data.names])
legacy_x = make_variant(bundle, "legacy")[order]
patch_x = make_variant(bundle, "meanmax_clspatchmax")[order]
folds = make_folds(data, seed)
legacy = oof_probabilities(
legacy_x, data.y, data.origins, data.anchor_idx, data.target_idx, folds,
Spec("legacy", 0.003, class_weight=None),
)
knn = knn_oof(patch_x, data.y, data.anchor_idx, data.target_idx, folds, k=11, temperature=0.1)
cache_dir.mkdir(parents=True, exist_ok=True)
np.savez(cache, names=data.names[data.target_idx], legacy=legacy, knn=knn)
return {"legacy": legacy, "knn": knn}
def logit(p: np.ndarray) -> np.ndarray:
p = np.clip(p, EPS, 1 - EPS)
return np.log(p / (1 - p))
def decode_crossfit(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
prior_weight: float,
cardinality_weight: float,
card_features: np.ndarray | None = None,
card_c: float = 1.0,
) -> np.ndarray:
"""v801 structured decoder (q=0); optional extra cardinality features."""
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y, labelsets)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
features = meta_features(probabilities)
if card_features is not None:
features = np.column_stack((features, card_features))
predictions = np.zeros_like(y)
logits = logit(probabilities)
for train, valid in folds:
counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64)
log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets)))
cardinality = LogisticRegression(
C=card_c, max_iter=3000, class_weight="balanced", random_state=801
).fit(features[train], y[train].sum(axis=1))
card_log = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1))
score = logits[valid] @ labelsets.T
score += prior_weight * log_prior
score += cardinality_weight * card_log[:, cardinalities]
predictions[valid] = labelsets[score.argmax(axis=1)]
return predictions
def evaluate(
data: Data,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
prior_weight: float = 0.5,
cardinality_weight: float = 0.25,
card_features: np.ndarray | None = None,
) -> dict[str, float]:
y = data.y[data.target_idx]
predictions = decode_crossfit(
y, probabilities, folds, prior_weight, cardinality_weight, card_features
)
metrics = report_metrics(y, probabilities, predictions)
recent = data.origins[data.target_idx] == "incremental100"
return {
"exact": float(metrics["exact_match_accuracy"]),
"recent_exact": float(accuracy_score(y[recent], predictions[recent])),
"micro_f1": float(metrics["micro_f1"]),
"map": float(metrics["mean_average_precision"]),
"card_acc": float((predictions.sum(1) == y.sum(1)).mean()),
}
def summarize(results: list[dict[str, float]]) -> dict[str, float]:
exact = [r["exact"] for r in results]
return {
"mean_exact": round(float(np.mean(exact)), 4),
"min_exact": round(float(np.min(exact)), 4),
"mean_recent": round(float(np.mean([r["recent_exact"] for r in results])), 4),
"mean_map": round(float(np.mean([r["map"] for r in results])), 4),
"mean_card": round(float(np.mean([r["card_acc"] for r in results])), 4),
}
def dump(path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
|