"""Patch-level multiple-instance (MIL) heads on frozen patch tokens for v802.""" from __future__ import annotations from dataclasses import asdict, dataclass import numpy as np import torch from torch import nn @dataclass(frozen=True) class MilConfig: hidden: int = 0 # 0 = linear patch scorer pooling: str = "lse" # lse | topk | attn r: float = 4.0 # LSE sharpness topk: int = 8 lr: float = 2e-3 weight_decay: float = 1e-2 epochs: int = 40 batch: int = 32 token_drop: float = 0.3 dropout: float = 0.1 card_head: bool = True # auxiliary 1/2/3 cardinality head card_loss: float = 0.5 seeds: int = 3 # torch-seed ensemble size def as_dict(self) -> dict[str, object]: return asdict(self) def fit_projection( patches: np.ndarray, fit_idx: np.ndarray, dims: int, seed: int = 802, sample: int = 120_000 ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Standardize + PCA fitted on patches of fit_idx images only.""" rng = np.random.default_rng(seed) per_image = max(1, sample // len(fit_idx)) rows = [] for index in fit_idx: take = rng.choice(patches.shape[1], size=min(per_image, patches.shape[1]), replace=False) rows.append(np.asarray(patches[index][np.sort(take)], dtype=np.float32)) x = np.concatenate(rows) mean = x.mean(0) std = x.std(0) + 1e-4 z = (x - mean) / std _, _, vt = np.linalg.svd(z[: min(len(z), 60_000)], full_matrices=False) return mean, std, vt[:dims].T.astype(np.float32) def project(patches: np.ndarray, mean: np.ndarray, std: np.ndarray, basis: np.ndarray) -> np.ndarray: out = np.empty((patches.shape[0], patches.shape[1], basis.shape[1]), dtype=np.float32) for start in range(0, len(patches), 50): chunk = np.asarray(patches[start : start + 50], dtype=np.float32) out[start : start + 50] = ((chunk - mean) / std) @ basis out /= np.sqrt(basis.shape[1]) return out class MilHead(nn.Module): def __init__(self, dims: int, classes: int, cfg: MilConfig): super().__init__() self.cfg = cfg width = cfg.hidden or dims self.body = ( nn.Sequential(nn.Dropout(cfg.dropout), nn.Linear(dims, cfg.hidden), nn.GELU(), nn.Dropout(cfg.dropout)) if cfg.hidden else nn.Dropout(cfg.dropout) ) self.score = nn.Linear(width, classes) self.attn = nn.Linear(width, classes) if cfg.pooling == "attn" else None self.card = nn.Linear(width + classes, 3) if cfg.card_head else None def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]: h = self.body(x) # (B, P, W) s = self.score(h) # (B, P, C) cfg = self.cfg if cfg.pooling == "lse": logits = torch.logsumexp(cfg.r * s, dim=1) / cfg.r - np.log(s.shape[1]) / cfg.r elif cfg.pooling == "topk": logits = s.topk(min(cfg.topk, s.shape[1]), dim=1).values.mean(1) else: a = torch.softmax(self.attn(h), dim=1) logits = (a * s).sum(1) card = None if self.card is not None: pooled = torch.cat((h.mean(1), torch.sigmoid(s).amax(1)), dim=1) card = self.card(pooled) return logits, card def train_predict( x: np.ndarray, y: np.ndarray, train_idx: np.ndarray, predict_idx: np.ndarray, cfg: MilConfig, seed: int, x_aug: np.ndarray | None = None, models_out: list | None = None, ) -> tuple[np.ndarray, np.ndarray]: """Returns (class probabilities, cardinality probabilities) averaged over seeds. x_aug (e.g. horizontally flipped views) doubles the training set and is averaged with x at prediction time (test-time augmentation). If models_out is given, each trained head's state_dict is appended to it. """ probs = np.zeros((len(predict_idx), y.shape[1])) cards = np.zeros((len(predict_idx), 3)) views = [x] if x_aug is None else [x, x_aug] xt = torch.from_numpy(np.concatenate([v[train_idx] for v in views])) yt = torch.from_numpy(np.tile(y[train_idx], (len(views), 1)).astype(np.float32)) ct = torch.from_numpy(np.tile(y[train_idx].sum(1) - 1, len(views)).astype(np.int64)) xps = [torch.from_numpy(v[predict_idx]) for v in views] for offset in range(cfg.seeds): torch.manual_seed(seed * 31 + offset) model = MilHead(x.shape[2], y.shape[1], cfg) opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay) steps = cfg.epochs * int(np.ceil(len(xt) / cfg.batch)) sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=cfg.lr, total_steps=steps, pct_start=0.1) generator = torch.Generator().manual_seed(seed * 31 + offset) model.train() for _ in range(cfg.epochs): order = torch.randperm(len(xt), generator=generator) for start in range(0, len(order), cfg.batch): batch = order[start : start + cfg.batch] xb = xt[batch] if cfg.token_drop > 0: keep = int(xb.shape[1] * (1 - cfg.token_drop)) choice = torch.rand(xb.shape[0], xb.shape[1], generator=generator).argsort(1)[:, :keep] xb = torch.gather(xb, 1, choice[..., None].expand(-1, -1, xb.shape[2])) logits, card = model(xb) loss = nn.functional.binary_cross_entropy_with_logits(logits, yt[batch]) if card is not None: loss = loss + cfg.card_loss * nn.functional.cross_entropy(card, ct[batch]) opt.zero_grad() loss.backward() opt.step() sched.step() model.eval() if models_out is not None: models_out.append({k: v.detach().clone() for k, v in model.state_dict().items()}) with torch.inference_mode(): for xp in xps: for start in range(0, len(xp), 64): logits, card = model(xp[start : start + 64]) probs[start : start + 64] += torch.sigmoid(logits).numpy() if card is not None: cards[start : start + 64] += torch.softmax(card, 1).numpy() return probs / (cfg.seeds * len(xps)), cards / (cfg.seeds * len(xps)) def mil_oof( x: np.ndarray, y: np.ndarray, anchor_idx: np.ndarray, target_idx: np.ndarray, folds: list[tuple[np.ndarray, np.ndarray]], cfg: MilConfig, seed: int, x_aug: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray]: probs = np.zeros((len(target_idx), y.shape[1])) cards = np.zeros((len(target_idx), 3)) for fold, (train_local, valid_local) in enumerate(folds): train_idx = np.concatenate((anchor_idx, target_idx[train_local])) p, c = train_predict(x, y, train_idx, target_idx[valid_local], cfg, seed + fold, x_aug) probs[valid_local] = p cards[valid_local] = c return probs, cards