File size: 1,901 Bytes
8fd3398
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
"""Next Shift: few-shot adaptation of a digit recogniser to a new visual environment.

Pieces shared by the notebook, inference.py and (ported line by line) web/core.js:
* conv features: nlrt layer lists exported from PyTorch (BatchNorm folded); `conv_target` uses BatchNorm
  statistics re-estimated on *unlabeled* target images (AdaBN), `conv_source` the source statistics.
* finetune_top: full-batch gradient descent on the top MLP (512 -> 64 -> 10) from the pretrained weights.
* train_linear: logistic-regression probe from zero weights on frozen 64-d source features.
All arithmetic is float64 and deterministic.
"""
import numpy as np

from nlrt import forward


def conv_features(layers, images):
    """images: (N, 16, 16) in [0, 1] -> (N, 512)."""
    return forward(layers, np.asarray(images, dtype=np.float64)[:, None, :, :])


def mlp_logits(p, F):
    h = np.maximum(F @ p["w1"].T + p["b1"], 0)
    return h @ p["w2"].T + p["b2"], h


def softmax(z):
    z = z - z.max(axis=1, keepdims=True)
    e = np.exp(z)
    return e / e.sum(axis=1, keepdims=True)


def finetune_top(p0, F, y, steps, lr):
    p = {k: np.array(v, dtype=np.float64) for k, v in p0.items()}
    n = len(y)
    for _ in range(steps):
        z, h = mlp_logits(p, F)
        P = softmax(z)
        P[np.arange(n), y] -= 1.0
        P /= n
        gw2 = P.T @ h
        gb2 = P.sum(axis=0)
        dh = (P @ p["w2"]) * (h > 0)
        gw1 = dh.T @ F
        gb1 = dh.sum(axis=0)
        p["w1"] -= lr * gw1
        p["b1"] -= lr * gb1
        p["w2"] -= lr * gw2
        p["b2"] -= lr * gb2
    return p


def train_linear(Z, y, steps, lr, n_classes=10):
    n, d = Z.shape
    W = np.zeros((n_classes, d))
    b = np.zeros(n_classes)
    for _ in range(steps):
        P = softmax(Z @ W.T + b)
        P[np.arange(n), y] -= 1.0
        P /= n
        W -= lr * (P.T @ Z)
        b -= lr * P.sum(axis=0)
    return W, b