v802-classifier / src /v802_mil.py
sneo's picture
Add v802 classifier head and source code
59839e3 verified
Raw History Blame Contribute Delete
7.08 kB
"""Patch-level multiple-instance (MIL) heads on frozen patch tokens for v802."""
from __future__ import annotations
from dataclasses import asdict, dataclass
import numpy as np
import torch
from torch import nn
@dataclass(frozen=True)
class MilConfig:
hidden: int = 0 # 0 = linear patch scorer
pooling: str = "lse" # lse | topk | attn
r: float = 4.0 # LSE sharpness
topk: int = 8
lr: float = 2e-3
weight_decay: float = 1e-2
epochs: int = 40
batch: int = 32
token_drop: float = 0.3
dropout: float = 0.1
card_head: bool = True # auxiliary 1/2/3 cardinality head
card_loss: float = 0.5
seeds: int = 3 # torch-seed ensemble size
def as_dict(self) -> dict[str, object]:
return asdict(self)
def fit_projection(
patches: np.ndarray, fit_idx: np.ndarray, dims: int, seed: int = 802, sample: int = 120_000
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Standardize + PCA fitted on patches of fit_idx images only."""
rng = np.random.default_rng(seed)
per_image = max(1, sample // len(fit_idx))
rows = []
for index in fit_idx:
take = rng.choice(patches.shape[1], size=min(per_image, patches.shape[1]), replace=False)
rows.append(np.asarray(patches[index][np.sort(take)], dtype=np.float32))
x = np.concatenate(rows)
mean = x.mean(0)
std = x.std(0) + 1e-4
z = (x - mean) / std
_, _, vt = np.linalg.svd(z[: min(len(z), 60_000)], full_matrices=False)
return mean, std, vt[:dims].T.astype(np.float32)
def project(patches: np.ndarray, mean: np.ndarray, std: np.ndarray, basis: np.ndarray) -> np.ndarray:
out = np.empty((patches.shape[0], patches.shape[1], basis.shape[1]), dtype=np.float32)
for start in range(0, len(patches), 50):
chunk = np.asarray(patches[start : start + 50], dtype=np.float32)
out[start : start + 50] = ((chunk - mean) / std) @ basis
out /= np.sqrt(basis.shape[1])
return out
class MilHead(nn.Module):
def __init__(self, dims: int, classes: int, cfg: MilConfig):
super().__init__()
self.cfg = cfg
width = cfg.hidden or dims
self.body = (
nn.Sequential(nn.Dropout(cfg.dropout), nn.Linear(dims, cfg.hidden), nn.GELU(), nn.Dropout(cfg.dropout))
if cfg.hidden else nn.Dropout(cfg.dropout)
)
self.score = nn.Linear(width, classes)
self.attn = nn.Linear(width, classes) if cfg.pooling == "attn" else None
self.card = nn.Linear(width + classes, 3) if cfg.card_head else None
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
h = self.body(x) # (B, P, W)
s = self.score(h) # (B, P, C)
cfg = self.cfg
if cfg.pooling == "lse":
logits = torch.logsumexp(cfg.r * s, dim=1) / cfg.r - np.log(s.shape[1]) / cfg.r
elif cfg.pooling == "topk":
logits = s.topk(min(cfg.topk, s.shape[1]), dim=1).values.mean(1)
else:
a = torch.softmax(self.attn(h), dim=1)
logits = (a * s).sum(1)
card = None
if self.card is not None:
pooled = torch.cat((h.mean(1), torch.sigmoid(s).amax(1)), dim=1)
card = self.card(pooled)
return logits, card
def train_predict(
x: np.ndarray,
y: np.ndarray,
train_idx: np.ndarray,
predict_idx: np.ndarray,
cfg: MilConfig,
seed: int,
x_aug: np.ndarray | None = None,
models_out: list | None = None,
) -> tuple[np.ndarray, np.ndarray]:
"""Returns (class probabilities, cardinality probabilities) averaged over seeds.
x_aug (e.g. horizontally flipped views) doubles the training set and is
averaged with x at prediction time (test-time augmentation). If models_out
is given, each trained head's state_dict is appended to it.
"""
probs = np.zeros((len(predict_idx), y.shape[1]))
cards = np.zeros((len(predict_idx), 3))
views = [x] if x_aug is None else [x, x_aug]
xt = torch.from_numpy(np.concatenate([v[train_idx] for v in views]))
yt = torch.from_numpy(np.tile(y[train_idx], (len(views), 1)).astype(np.float32))
ct = torch.from_numpy(np.tile(y[train_idx].sum(1) - 1, len(views)).astype(np.int64))
xps = [torch.from_numpy(v[predict_idx]) for v in views]
for offset in range(cfg.seeds):
torch.manual_seed(seed * 31 + offset)
model = MilHead(x.shape[2], y.shape[1], cfg)
opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
steps = cfg.epochs * int(np.ceil(len(xt) / cfg.batch))
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=cfg.lr, total_steps=steps, pct_start=0.1)
generator = torch.Generator().manual_seed(seed * 31 + offset)
model.train()
for _ in range(cfg.epochs):
order = torch.randperm(len(xt), generator=generator)
for start in range(0, len(order), cfg.batch):
batch = order[start : start + cfg.batch]
xb = xt[batch]
if cfg.token_drop > 0:
keep = int(xb.shape[1] * (1 - cfg.token_drop))
choice = torch.rand(xb.shape[0], xb.shape[1], generator=generator).argsort(1)[:, :keep]
xb = torch.gather(xb, 1, choice[..., None].expand(-1, -1, xb.shape[2]))
logits, card = model(xb)
loss = nn.functional.binary_cross_entropy_with_logits(logits, yt[batch])
if card is not None:
loss = loss + cfg.card_loss * nn.functional.cross_entropy(card, ct[batch])
opt.zero_grad()
loss.backward()
opt.step()
sched.step()
model.eval()
if models_out is not None:
models_out.append({k: v.detach().clone() for k, v in model.state_dict().items()})
with torch.inference_mode():
for xp in xps:
for start in range(0, len(xp), 64):
logits, card = model(xp[start : start + 64])
probs[start : start + 64] += torch.sigmoid(logits).numpy()
if card is not None:
cards[start : start + 64] += torch.softmax(card, 1).numpy()
return probs / (cfg.seeds * len(xps)), cards / (cfg.seeds * len(xps))
def mil_oof(
x: np.ndarray,
y: np.ndarray,
anchor_idx: np.ndarray,
target_idx: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
cfg: MilConfig,
seed: int,
x_aug: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray]:
probs = np.zeros((len(target_idx), y.shape[1]))
cards = np.zeros((len(target_idx), 3))
for fold, (train_local, valid_local) in enumerate(folds):
train_idx = np.concatenate((anchor_idx, target_idx[train_local]))
p, c = train_predict(x, y, train_idx, target_idx[valid_local], cfg, seed + fold, x_aug)
probs[valid_local] = p
cards[valid_local] = c
return probs, cards