File size: 7,083 Bytes
59839e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
"""Patch-level multiple-instance (MIL) heads on frozen patch tokens for v802."""

from __future__ import annotations

from dataclasses import asdict, dataclass

import numpy as np
import torch
from torch import nn


@dataclass(frozen=True)
class MilConfig:
    hidden: int = 0            # 0 = linear patch scorer
    pooling: str = "lse"       # lse | topk | attn
    r: float = 4.0             # LSE sharpness
    topk: int = 8
    lr: float = 2e-3
    weight_decay: float = 1e-2
    epochs: int = 40
    batch: int = 32
    token_drop: float = 0.3
    dropout: float = 0.1
    card_head: bool = True     # auxiliary 1/2/3 cardinality head
    card_loss: float = 0.5
    seeds: int = 3             # torch-seed ensemble size

    def as_dict(self) -> dict[str, object]:
        return asdict(self)


def fit_projection(
    patches: np.ndarray, fit_idx: np.ndarray, dims: int, seed: int = 802, sample: int = 120_000
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Standardize + PCA fitted on patches of fit_idx images only."""
    rng = np.random.default_rng(seed)
    per_image = max(1, sample // len(fit_idx))
    rows = []
    for index in fit_idx:
        take = rng.choice(patches.shape[1], size=min(per_image, patches.shape[1]), replace=False)
        rows.append(np.asarray(patches[index][np.sort(take)], dtype=np.float32))
    x = np.concatenate(rows)
    mean = x.mean(0)
    std = x.std(0) + 1e-4
    z = (x - mean) / std
    _, _, vt = np.linalg.svd(z[: min(len(z), 60_000)], full_matrices=False)
    return mean, std, vt[:dims].T.astype(np.float32)


def project(patches: np.ndarray, mean: np.ndarray, std: np.ndarray, basis: np.ndarray) -> np.ndarray:
    out = np.empty((patches.shape[0], patches.shape[1], basis.shape[1]), dtype=np.float32)
    for start in range(0, len(patches), 50):
        chunk = np.asarray(patches[start : start + 50], dtype=np.float32)
        out[start : start + 50] = ((chunk - mean) / std) @ basis
    out /= np.sqrt(basis.shape[1])
    return out


class MilHead(nn.Module):
    def __init__(self, dims: int, classes: int, cfg: MilConfig):
        super().__init__()
        self.cfg = cfg
        width = cfg.hidden or dims
        self.body = (
            nn.Sequential(nn.Dropout(cfg.dropout), nn.Linear(dims, cfg.hidden), nn.GELU(), nn.Dropout(cfg.dropout))
            if cfg.hidden else nn.Dropout(cfg.dropout)
        )
        self.score = nn.Linear(width, classes)
        self.attn = nn.Linear(width, classes) if cfg.pooling == "attn" else None
        self.card = nn.Linear(width + classes, 3) if cfg.card_head else None

    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]:
        h = self.body(x)                       # (B, P, W)
        s = self.score(h)                      # (B, P, C)
        cfg = self.cfg
        if cfg.pooling == "lse":
            logits = torch.logsumexp(cfg.r * s, dim=1) / cfg.r - np.log(s.shape[1]) / cfg.r
        elif cfg.pooling == "topk":
            logits = s.topk(min(cfg.topk, s.shape[1]), dim=1).values.mean(1)
        else:
            a = torch.softmax(self.attn(h), dim=1)
            logits = (a * s).sum(1)
        card = None
        if self.card is not None:
            pooled = torch.cat((h.mean(1), torch.sigmoid(s).amax(1)), dim=1)
            card = self.card(pooled)
        return logits, card


def train_predict(
    x: np.ndarray,
    y: np.ndarray,
    train_idx: np.ndarray,
    predict_idx: np.ndarray,
    cfg: MilConfig,
    seed: int,
    x_aug: np.ndarray | None = None,
    models_out: list | None = None,
) -> tuple[np.ndarray, np.ndarray]:
    """Returns (class probabilities, cardinality probabilities) averaged over seeds.

    x_aug (e.g. horizontally flipped views) doubles the training set and is
    averaged with x at prediction time (test-time augmentation). If models_out
    is given, each trained head's state_dict is appended to it.
    """
    probs = np.zeros((len(predict_idx), y.shape[1]))
    cards = np.zeros((len(predict_idx), 3))
    views = [x] if x_aug is None else [x, x_aug]
    xt = torch.from_numpy(np.concatenate([v[train_idx] for v in views]))
    yt = torch.from_numpy(np.tile(y[train_idx], (len(views), 1)).astype(np.float32))
    ct = torch.from_numpy(np.tile(y[train_idx].sum(1) - 1, len(views)).astype(np.int64))
    xps = [torch.from_numpy(v[predict_idx]) for v in views]
    for offset in range(cfg.seeds):
        torch.manual_seed(seed * 31 + offset)
        model = MilHead(x.shape[2], y.shape[1], cfg)
        opt = torch.optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)
        steps = cfg.epochs * int(np.ceil(len(xt) / cfg.batch))
        sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=cfg.lr, total_steps=steps, pct_start=0.1)
        generator = torch.Generator().manual_seed(seed * 31 + offset)
        model.train()
        for _ in range(cfg.epochs):
            order = torch.randperm(len(xt), generator=generator)
            for start in range(0, len(order), cfg.batch):
                batch = order[start : start + cfg.batch]
                xb = xt[batch]
                if cfg.token_drop > 0:
                    keep = int(xb.shape[1] * (1 - cfg.token_drop))
                    choice = torch.rand(xb.shape[0], xb.shape[1], generator=generator).argsort(1)[:, :keep]
                    xb = torch.gather(xb, 1, choice[..., None].expand(-1, -1, xb.shape[2]))
                logits, card = model(xb)
                loss = nn.functional.binary_cross_entropy_with_logits(logits, yt[batch])
                if card is not None:
                    loss = loss + cfg.card_loss * nn.functional.cross_entropy(card, ct[batch])
                opt.zero_grad()
                loss.backward()
                opt.step()
                sched.step()
        model.eval()
        if models_out is not None:
            models_out.append({k: v.detach().clone() for k, v in model.state_dict().items()})
        with torch.inference_mode():
            for xp in xps:
                for start in range(0, len(xp), 64):
                    logits, card = model(xp[start : start + 64])
                    probs[start : start + 64] += torch.sigmoid(logits).numpy()
                    if card is not None:
                        cards[start : start + 64] += torch.softmax(card, 1).numpy()
    return probs / (cfg.seeds * len(xps)), cards / (cfg.seeds * len(xps))


def mil_oof(
    x: np.ndarray,
    y: np.ndarray,
    anchor_idx: np.ndarray,
    target_idx: np.ndarray,
    folds: list[tuple[np.ndarray, np.ndarray]],
    cfg: MilConfig,
    seed: int,
    x_aug: np.ndarray | None = None,
) -> tuple[np.ndarray, np.ndarray]:
    probs = np.zeros((len(target_idx), y.shape[1]))
    cards = np.zeros((len(target_idx), 3))
    for fold, (train_local, valid_local) in enumerate(folds):
        train_idx = np.concatenate((anchor_idx, target_idx[train_local]))
        p, c = train_predict(x, y, train_idx, target_idx[valid_local], cfg, seed + fold, x_aug)
        probs[valid_local] = p
        cards[valid_local] = c
    return probs, cards