File size: 7,083 Bytes
59839e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Patch-level multiple-instance (MIL) heads on frozen patch tokens for v802."""
from __future__ import annotations
from dataclasses import asdict, dataclass
import numpy as np
import torch
from torch import nn
@dataclass(frozen=True)
class MilConfig:
hidden: int = 0 # 0 = linear patch scorer
pooling: str = "lse" # lse | topk | attn
r: float = 4.0 # LSE sharpness
topk: int = 8
lr: float = 2e-3
weight_decay: float = 1e-2
epochs: int = 40
batch: int = 32
token_drop: float = 0.3
dropout: float = 0.1
card_head: bool = True # auxiliary 1/2/3 cardinality head
card_loss: float = 0.5
seeds: int = 3 # torch-seed ensemble size
def as_dict(self) -> dict[str, object]:
return asdict(self)
def fit_projection(
patches: np.ndarray, fit_idx: np.ndarray, dims: int, seed: int = 802, sample: int = 120_000
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Standardize + PCA fitted on patches of fit_idx images only."""
rng = np.random.default_rng(seed)
per_image = max(1, sample // len(fit_idx))
rows = []
for index in fit_idx:
take = rng.choice(patches.shape[1], size=min(per_image, patches.shape[1]), replace=False)
rows.append(np.asarray(patches[index][np.sort(take)], dtype=np.float32))
x = np.concatenate(rows)
mean = x.mean(0)
std = x.std(0) + 1e-4
z = (x - mean) / std
_, _, vt = np.linalg.svd(z[: min(len(z), 60_000)], full_matrices=False)
return mean, std, vt[:dims].T.astype(np.float32)
def project(patches: np.ndarray, mean: np.ndarray, std: np.ndarray, basis: np.ndarray) -> np.ndarray:
out = np.empty((patches.shape[0], patches.shape[1], basis.shape[1]), dtype=np.float32)
for start in range(0, len(patches), 50):
chunk = np.asarray(patches[start : start + 50], dtype=np.float32)
out[start : start + 50] = ((chunk - mean) / std) @ basis
out /= np.sqrt(basis.shape[1])
return out
class MilHead(nn.Module):
def __init__(self, dims: int, classes: int, cfg: MilConfig):
super().__init__()
self.cfg = cfg
width = cfg.hidden or dims
self.body = (
nn.Sequential(nn.Dropout(cfg.dropout), nn.Linear(dims, cfg.hidden), nn.GELU(), nn.Dropout(cfg.dropout))
if cfg.hidden else nn.Dropout(cfg.dropout)
)
self.score = nn.Linear(width, classes)
self.attn = nn.Linear(width, classes) if cfg.pooling == "attn" else None
self.card = nn.Linear(width + classes, 3) if cfg.card_head else None
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
h = self.body(x) # (B, P, W)
s = self.score(h) # (B, P, C)
cfg = self.cfg
if cfg.pooling == "lse":
logits = torch.logsumexp(cfg.r * s, dim=1) / cfg.r - np.log(s.shape[1]) / cfg.r
elif cfg.pooling == "topk":
logits = s.topk(min(cfg.topk, s.shape[1]), dim=1).values.mean(1)
else:
a = torch.softmax(self.attn(h), dim=1)
logits = (a * s).sum(1)
card = None
if self.card is not None:
pooled = torch.cat((h.mean(1), torch.sigmoid(s).amax(1)), dim=1)
card = self.card(pooled)
return logits, card
def train_predict(
x: np.ndarray,
y: np.ndarray,
train_idx: np.ndarray,
predict_idx: np.ndarray,
cfg: MilConfig,
seed: int,
x_aug: np.ndarray | None = None,
models_out: list | None = None,
) -> tuple[np.ndarray, np.ndarray]:
"""Returns (class probabilities, cardinality probabilities) averaged over seeds.
x_aug (e.g. horizontally flipped views) doubles the training set and is
averaged with x at prediction time (test-time augmentation). If models_out
is given, each trained head's state_dict is appended to it.
"""
probs = np.zeros((len(predict_idx), y.shape[1]))
cards = np.zeros((len(predict_idx), 3))
views = [x] if x_aug is None else [x, x_aug]
xt = torch.from_numpy(np.concatenate([v[train_idx] for v in views]))
yt = torch.from_numpy(np.tile(y[train_idx], (len(views), 1)).astype(np.float32))
ct = torch.from_numpy(np.tile(y[train_idx].sum(1) - 1, len(views)).astype(np.int64))
xps = [torch.from_numpy(v[predict_idx]) for v in views]
for offset in range(cfg.seeds):
torch.manual_seed(seed * 31 + offset)
model = MilHead(x.shape[2], y.shape[1], cfg)
opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
steps = cfg.epochs * int(np.ceil(len(xt) / cfg.batch))
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=cfg.lr, total_steps=steps, pct_start=0.1)
generator = torch.Generator().manual_seed(seed * 31 + offset)
model.train()
for _ in range(cfg.epochs):
order = torch.randperm(len(xt), generator=generator)
for start in range(0, len(order), cfg.batch):
batch = order[start : start + cfg.batch]
xb = xt[batch]
if cfg.token_drop > 0:
keep = int(xb.shape[1] * (1 - cfg.token_drop))
choice = torch.rand(xb.shape[0], xb.shape[1], generator=generator).argsort(1)[:, :keep]
xb = torch.gather(xb, 1, choice[..., None].expand(-1, -1, xb.shape[2]))
logits, card = model(xb)
loss = nn.functional.binary_cross_entropy_with_logits(logits, yt[batch])
if card is not None:
loss = loss + cfg.card_loss * nn.functional.cross_entropy(card, ct[batch])
opt.zero_grad()
loss.backward()
opt.step()
sched.step()
model.eval()
if models_out is not None:
models_out.append({k: v.detach().clone() for k, v in model.state_dict().items()})
with torch.inference_mode():
for xp in xps:
for start in range(0, len(xp), 64):
logits, card = model(xp[start : start + 64])
probs[start : start + 64] += torch.sigmoid(logits).numpy()
if card is not None:
cards[start : start + 64] += torch.softmax(card, 1).numpy()
return probs / (cfg.seeds * len(xps)), cards / (cfg.seeds * len(xps))
def mil_oof(
x: np.ndarray,
y: np.ndarray,
anchor_idx: np.ndarray,
target_idx: np.ndarray,
folds: list[tuple[np.ndarray, np.ndarray]],
cfg: MilConfig,
seed: int,
x_aug: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray]:
probs = np.zeros((len(target_idx), y.shape[1]))
cards = np.zeros((len(target_idx), 3))
for fold, (train_local, valid_local) in enumerate(folds):
train_idx = np.concatenate((anchor_idx, target_idx[train_local]))
p, c = train_predict(x, y, train_idx, target_idx[valid_local], cfg, seed + fold, x_aug)
probs[valid_local] = p
cards[valid_local] = c
return probs, cards
|