Download src/v802_mil.py from sneo/v802-classifier: direct link, hf CLI and curl.
- Browser
- Download file 7.08 kB
-
https://huggingface.co/sneo/v802-classifier/resolve/main/src/v802_mil.py
- Command line
-
hf download hf://sneo/v802-classifier/src/v802_mil.py
-
curl -L -o v802_mil.py https://huggingface.co/sneo/v802-classifier/resolve/main/src/v802_mil.py
7.08 kB
| """Patch-level multiple-instance (MIL) heads on frozen patch tokens for v802.""" | |
| from __future__ import annotations | |
| from dataclasses import asdict, dataclass | |
| import numpy as np | |
| import torch | |
| from torch import nn | |
| class MilConfig: | |
| hidden: int = 0 # 0 = linear patch scorer | |
| pooling: str = "lse" # lse | topk | attn | |
| r: float = 4.0 # LSE sharpness | |
| topk: int = 8 | |
| lr: float = 2e-3 | |
| weight_decay: float = 1e-2 | |
| epochs: int = 40 | |
| batch: int = 32 | |
| token_drop: float = 0.3 | |
| dropout: float = 0.1 | |
| card_head: bool = True # auxiliary 1/2/3 cardinality head | |
| card_loss: float = 0.5 | |
| seeds: int = 3 # torch-seed ensemble size | |
| def as_dict(self) -> dict[str, object]: | |
| return asdict(self) | |
| def fit_projection( | |
| patches: np.ndarray, fit_idx: np.ndarray, dims: int, seed: int = 802, sample: int = 120_000 | |
| ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: | |
| """Standardize + PCA fitted on patches of fit_idx images only.""" | |
| rng = np.random.default_rng(seed) | |
| per_image = max(1, sample // len(fit_idx)) | |
| rows = [] | |
| for index in fit_idx: | |
| take = rng.choice(patches.shape[1], size=min(per_image, patches.shape[1]), replace=False) | |
| rows.append(np.asarray(patches[index][np.sort(take)], dtype=np.float32)) | |
| x = np.concatenate(rows) | |
| mean = x.mean(0) | |
| std = x.std(0) + 1e-4 | |
| z = (x - mean) / std | |
| _, _, vt = np.linalg.svd(z[: min(len(z), 60_000)], full_matrices=False) | |
| return mean, std, vt[:dims].T.astype(np.float32) | |
| def project(patches: np.ndarray, mean: np.ndarray, std: np.ndarray, basis: np.ndarray) -> np.ndarray: | |
| out = np.empty((patches.shape[0], patches.shape[1], basis.shape[1]), dtype=np.float32) | |
| for start in range(0, len(patches), 50): | |
| chunk = np.asarray(patches[start : start + 50], dtype=np.float32) | |
| out[start : start + 50] = ((chunk - mean) / std) @ basis | |
| out /= np.sqrt(basis.shape[1]) | |
| return out | |
| class MilHead(nn.Module): | |
| def __init__(self, dims: int, classes: int, cfg: MilConfig): | |
| super().__init__() | |
| self.cfg = cfg | |
| width = cfg.hidden or dims | |
| self.body = ( | |
| nn.Sequential(nn.Dropout(cfg.dropout), nn.Linear(dims, cfg.hidden), nn.GELU(), nn.Dropout(cfg.dropout)) | |
| if cfg.hidden else nn.Dropout(cfg.dropout) | |
| ) | |
| self.score = nn.Linear(width, classes) | |
| self.attn = nn.Linear(width, classes) if cfg.pooling == "attn" else None | |
| self.card = nn.Linear(width + classes, 3) if cfg.card_head else None | |
| def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]: | |
| h = self.body(x) # (B, P, W) | |
| s = self.score(h) # (B, P, C) | |
| cfg = self.cfg | |
| if cfg.pooling == "lse": | |
| logits = torch.logsumexp(cfg.r * s, dim=1) / cfg.r - np.log(s.shape[1]) / cfg.r | |
| elif cfg.pooling == "topk": | |
| logits = s.topk(min(cfg.topk, s.shape[1]), dim=1).values.mean(1) | |
| else: | |
| a = torch.softmax(self.attn(h), dim=1) | |
| logits = (a * s).sum(1) | |
| card = None | |
| if self.card is not None: | |
| pooled = torch.cat((h.mean(1), torch.sigmoid(s).amax(1)), dim=1) | |
| card = self.card(pooled) | |
| return logits, card | |
| def train_predict( | |
| x: np.ndarray, | |
| y: np.ndarray, | |
| train_idx: np.ndarray, | |
| predict_idx: np.ndarray, | |
| cfg: MilConfig, | |
| seed: int, | |
| x_aug: np.ndarray | None = None, | |
| models_out: list | None = None, | |
| ) -> tuple[np.ndarray, np.ndarray]: | |
| """Returns (class probabilities, cardinality probabilities) averaged over seeds. | |
| x_aug (e.g. horizontally flipped views) doubles the training set and is | |
| averaged with x at prediction time (test-time augmentation). If models_out | |
| is given, each trained head's state_dict is appended to it. | |
| """ | |
| probs = np.zeros((len(predict_idx), y.shape[1])) | |
| cards = np.zeros((len(predict_idx), 3)) | |
| views = [x] if x_aug is None else [x, x_aug] | |
| xt = torch.from_numpy(np.concatenate([v[train_idx] for v in views])) | |
| yt = torch.from_numpy(np.tile(y[train_idx], (len(views), 1)).astype(np.float32)) | |
| ct = torch.from_numpy(np.tile(y[train_idx].sum(1) - 1, len(views)).astype(np.int64)) | |
| xps = [torch.from_numpy(v[predict_idx]) for v in views] | |
| for offset in range(cfg.seeds): | |
| torch.manual_seed(seed * 31 + offset) | |
| model = MilHead(x.shape[2], y.shape[1], cfg) | |
| opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay) | |
| steps = cfg.epochs * int(np.ceil(len(xt) / cfg.batch)) | |
| sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=cfg.lr, total_steps=steps, pct_start=0.1) | |
| generator = torch.Generator().manual_seed(seed * 31 + offset) | |
| model.train() | |
| for _ in range(cfg.epochs): | |
| order = torch.randperm(len(xt), generator=generator) | |
| for start in range(0, len(order), cfg.batch): | |
| batch = order[start : start + cfg.batch] | |
| xb = xt[batch] | |
| if cfg.token_drop > 0: | |
| keep = int(xb.shape[1] * (1 - cfg.token_drop)) | |
| choice = torch.rand(xb.shape[0], xb.shape[1], generator=generator).argsort(1)[:, :keep] | |
| xb = torch.gather(xb, 1, choice[..., None].expand(-1, -1, xb.shape[2])) | |
| logits, card = model(xb) | |
| loss = nn.functional.binary_cross_entropy_with_logits(logits, yt[batch]) | |
| if card is not None: | |
| loss = loss + cfg.card_loss * nn.functional.cross_entropy(card, ct[batch]) | |
| opt.zero_grad() | |
| loss.backward() | |
| opt.step() | |
| sched.step() | |
| model.eval() | |
| if models_out is not None: | |
| models_out.append({k: v.detach().clone() for k, v in model.state_dict().items()}) | |
| with torch.inference_mode(): | |
| for xp in xps: | |
| for start in range(0, len(xp), 64): | |
| logits, card = model(xp[start : start + 64]) | |
| probs[start : start + 64] += torch.sigmoid(logits).numpy() | |
| if card is not None: | |
| cards[start : start + 64] += torch.softmax(card, 1).numpy() | |
| return probs / (cfg.seeds * len(xps)), cards / (cfg.seeds * len(xps)) | |
| def mil_oof( | |
| x: np.ndarray, | |
| y: np.ndarray, | |
| anchor_idx: np.ndarray, | |
| target_idx: np.ndarray, | |
| folds: list[tuple[np.ndarray, np.ndarray]], | |
| cfg: MilConfig, | |
| seed: int, | |
| x_aug: np.ndarray | None = None, | |
| ) -> tuple[np.ndarray, np.ndarray]: | |
| probs = np.zeros((len(target_idx), y.shape[1])) | |
| cards = np.zeros((len(target_idx), 3)) | |
| for fold, (train_local, valid_local) in enumerate(folds): | |
| train_idx = np.concatenate((anchor_idx, target_idx[train_local])) | |
| p, c = train_predict(x, y, train_idx, target_idx[valid_local], cfg, seed + fold, x_aug) | |
| probs[valid_local] = p | |
| cards[valid_local] = c | |
| return probs, cards | |