v802-classifier / src /v802_common.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
7.11 kB
"""Shared data, fold, decoder and baseline helpers for v802 experiments."""
from __future__ import annotations
import csv
import json
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from experiment_v801_knn import knn_oof
from train_v801 import (
EPS,
Spec,
all_labelsets,
labelset_ids,
make_variant,
meta_features,
oof_probabilities,
report_metrics,
)
ROOT = Path(__file__).resolve().parents[1]
SPLITS = ROOT / "artifacts" / "splits" / "v800"
LEGACY_FEATURES = ROOT / "artifacts" / "features" / "dinov3-vitl16-all800-v801.npz"
DEV_SEEDS = (801208, 801209, 801210, 801211)
CHECK_SEEDS = (802001, 802002, 802003)
V801_DECODER = {"prior_weight": 0.5, "cardinality_weight": 0.25}
def read_split(name: str) -> list[dict[str, str]]:
with (SPLITS / name).open(newline="", encoding="utf-8-sig") as handle:
return list(csv.DictReader(handle))
@dataclass
class Data:
names: np.ndarray
y: np.ndarray
classes: tuple[str, ...]
origins: np.ndarray
development_idx: np.ndarray
anchor_idx: np.ndarray
target_idx: np.ndarray
new_idx: np.ndarray
historical_idx: np.ndarray
def load_data(names: list[str] | np.ndarray) -> Data:
"""Index splits against a feature file order (list of file names)."""
names = np.asarray(names).astype(str)
lookup = {name: index for index, name in enumerate(names)}
bundle = np.load(LEGACY_FEATURES)
legacy_lookup = {n: i for i, n in enumerate(bundle["file_names"].astype(str))}
y_legacy = bundle["labels"].astype(np.uint8)
y = np.stack([y_legacy[legacy_lookup[name]] for name in names])
rows = read_split("development736.csv")
development_idx = np.asarray([lookup[row["file_name"]] for row in rows])
origins = np.full(len(names), "excluded", dtype="U32")
for row in rows:
origins[lookup[row["file_name"]]] = row["origin"]
anchor_idx = development_idx[origins[development_idx] == "old440"]
target_idx = development_idx[origins[development_idx] != "old440"]
new_idx = np.asarray([lookup[r["file_name"]] for r in read_split("new_holdout32.csv")])
historical_idx = np.asarray(
[lookup[r["file_name"]] for r in read_split("historical_holdout32.csv")]
)
assert (len(development_idx), len(anchor_idx), len(target_idx)) == (736, 440, 296)
return Data(
names, y, tuple(bundle["classes"].astype(str)), origins, development_idx,
anchor_idx, target_idx, new_idx, historical_idx,
)
def make_folds(data: Data, seed: int) -> list[tuple[np.ndarray, np.ndarray]]:
return list(
MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=seed).split(
np.zeros(len(data.target_idx)), data.y[data.target_idx]
)
)
def v801_oof(data: Data, seed: int, cache_dir: Path) -> dict[str, np.ndarray]:
"""Re-run the v801 legacy probe + kNN components on one seed's folds."""
cache = cache_dir / f"v801_oof_seed{seed}.npz"
if cache.exists():
stored = np.load(cache)
assert (stored["names"] == data.names[data.target_idx]).all()
return {"legacy": stored["legacy"], "knn": stored["knn"]}
bundle = np.load(LEGACY_FEATURES)
legacy_names = bundle["file_names"].astype(str)
order = np.asarray([{n: i for i, n in enumerate(legacy_names)}[n] for n in data.names])
legacy_x = make_variant(bundle, "legacy")[order]
patch_x = make_variant(bundle, "meanmax_clspatchmax")[order]
folds = make_folds(data, seed)
legacy = oof_probabilities(
legacy_x, data.y, data.origins, data.anchor_idx, data.target_idx, folds,
Spec("legacy", 0.003, class_weight=None),
)
knn = knn_oof(patch_x, data.y, data.anchor_idx, data.target_idx, folds, k=11, temperature=0.1)
cache_dir.mkdir(parents=True, exist_ok=True)
np.savez(cache, names=data.names[data.target_idx], legacy=legacy, knn=knn)
return {"legacy": legacy, "knn": knn}
def logit(p: np.ndarray) -> np.ndarray:
p = np.clip(p, EPS, 1 - EPS)
return np.log(p / (1 - p))
def decode_crossfit(
y: np.ndarray,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
prior_weight: float,
cardinality_weight: float,
card_features: np.ndarray | None = None,
card_c: float = 1.0,
) -> np.ndarray:
"""v801 structured decoder (q=0); optional extra cardinality features."""
labelsets = all_labelsets(y.shape[1])
set_ids = labelset_ids(y, labelsets)
cardinalities = labelsets.sum(axis=1).astype(int) - 1
features = meta_features(probabilities)
if card_features is not None:
features = np.column_stack((features, card_features))
predictions = np.zeros_like(y)
logits = logit(probabilities)
for train, valid in folds:
counts = np.bincount(set_ids[train], minlength=len(labelsets)).astype(np.float64)
log_prior = np.log((counts + 1.0) / (len(train) + len(labelsets)))
cardinality = LogisticRegression(
C=card_c, max_iter=3000, class_weight="balanced", random_state=801
).fit(features[train], y[train].sum(axis=1))
card_log = np.log(np.clip(cardinality.predict_proba(features[valid]), EPS, 1))
score = logits[valid] @ labelsets.T
score += prior_weight * log_prior
score += cardinality_weight * card_log[:, cardinalities]
predictions[valid] = labelsets[score.argmax(axis=1)]
return predictions
def evaluate(
data: Data,
probabilities: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
prior_weight: float = 0.5,
cardinality_weight: float = 0.25,
card_features: np.ndarray | None = None,
) -> dict[str, float]:
y = data.y[data.target_idx]
predictions = decode_crossfit(
y, probabilities, folds, prior_weight, cardinality_weight, card_features
)
metrics = report_metrics(y, probabilities, predictions)
recent = data.origins[data.target_idx] == "incremental100"
return {
"exact": float(metrics["exact_match_accuracy"]),
"recent_exact": float(accuracy_score(y[recent], predictions[recent])),
"micro_f1": float(metrics["micro_f1"]),
"map": float(metrics["mean_average_precision"]),
"card_acc": float((predictions.sum(1) == y.sum(1)).mean()),
}
def summarize(results: list[dict[str, float]]) -> dict[str, float]:
exact = [r["exact"] for r in results]
return {
"mean_exact": round(float(np.mean(exact)), 4),
"min_exact": round(float(np.min(exact)), 4),
"mean_recent": round(float(np.mean([r["recent_exact"] for r in results])), 4),
"mean_map": round(float(np.mean([r["map"] for r in results])), 4),
"mean_card": round(float(np.mean([r["card_acc"] for r in results])), 4),
}
def dump(path: Path, payload: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")