File size: 5,534 Bytes
db32cff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""Independent implementation of MoNo-light, paper equations (1)--(6), (29)--(30).

No author repository code is included. See .trainer_work/darcy85/source_capture.md.
Identical Darcy coordinates share one geometry route across a batch. All transport
iterations remain differentiable, including both encoder and decoder projections.
"""
import copy
import math

import torch
from torch import nn
from torch.nn import functional as F


class ResidualProjector(nn.Module):
    def __init__(self, inputs, width, outputs, dropout=0.0):
        super().__init__()
        self.lift = nn.Linear(inputs, width)
        self.hidden = nn.ModuleList(nn.Linear(width, width) for _ in range(3))
        self.drop = nn.Dropout(dropout)
        self.readout = nn.Linear(width, outputs)

    def forward(self, x):
        x = F.gelu(self.lift(x))
        for layer in self.hidden:
            x = x + self.drop(F.gelu(layer(x)))
        return self.readout(x)


def sinkhorn_transport(scores, iterations=8, temperature=1.0):
    """Uniform-marginal log Sinkhorn; finite iterations, never detached."""
    z = scores / temperature
    n, m = z.shape[-2:]
    v = z.new_zeros((*z.shape[:-2], 1, m))
    for _ in range(iterations):
        u = -math.log(n) - torch.logsumexp(z + v, dim=-1, keepdim=True)
        v = -math.log(m) - torch.logsumexp(z + u, dim=-2, keepdim=True)
    return torch.exp(z + u + v)


class TransformerLayer(nn.Module):
    def __init__(self, width, heads):
        super().__init__()
        self.heads = heads
        self.norm_attention = nn.LayerNorm(width)
        self.qkv = nn.Linear(width, width * 3)
        self.attention_out = nn.Linear(width, width)
        self.norm_ffn = nn.LayerNorm(width)
        self.ffn = nn.Sequential(nn.Linear(width, 4 * width), nn.GELU(),
                                 nn.Linear(4 * width, width))

    def forward(self, x):
        b, n, d = x.shape
        q, k, v = self.qkv(self.norm_attention(x)).reshape(
            b, n, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4).unbind(0)
        a = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
        x = x + self.attention_out(a.transpose(1, 2).reshape(b, n, d))
        return x + self.ffn(self.norm_ffn(x))


def initialize_readout(module):
    if isinstance(module, nn.Linear):
        nn.init.trunc_normal_(module.weight, std=0.02)
        if module.bias is not None:
            nn.init.zeros_(module.bias)
    elif isinstance(module, nn.LayerNorm):
        nn.init.ones_(module.weight)
        nn.init.zeros_(module.bias)


class MoNoLight(nn.Module):
    def __init__(self, width=96, latent_sizes=(512, 256, 128, 64),
                 depths=(3, 1, 1, 1), heads=(4, 4, 8, 8),
                 sinkhorn_iterations=8, temperature=1.0,
                 anchor_grad_fraction=0.25, output_dropout=0.5):
        super().__init__()
        self.latent_sizes = tuple(latent_sizes)
        self.iterations = sinkhorn_iterations
        self.temperature = temperature
        self.anchor_grad_fraction = anchor_grad_fraction
        self.anchor = ResidualProjector(2, width, width)
        self.frozen_anchor = copy.deepcopy(self.anchor).requires_grad_(False)
        self.physical = ResidualProjector(3, width, width)
        self.assignment = nn.ModuleList(
            ResidualProjector(width, width, n) for n in latent_sizes)
        self.encoder = nn.ModuleList(nn.Sequential(*[
            TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads))
        self.decoder = nn.ModuleList(nn.Sequential(*[
            TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads))
        self.output = ResidualProjector(width, width, 1, output_dropout)
        self.encoder.apply(initialize_readout)
        self.decoder.apply(initialize_readout)
        self.output.apply(initialize_readout)

    def embed_geometry(self, coordinates):
        if not self.training:
            return self.frozen_anchor(coordinates)
        # Synchronize before the train forward, as documented in the existing spec.
        with torch.no_grad():
            for dst, src in zip(self.frozen_anchor.parameters(), self.anchor.parameters()):
                dst.copy_(src)
        n = coordinates.shape[-2]
        chosen = torch.randperm(n, device=coordinates.device)[:math.ceil(n * self.anchor_grad_fraction)]
        # Evaluate the frozen copy everywhere; replace chosen points by trainable outputs.
        with torch.no_grad():
            g = self.frozen_anchor(coordinates)
        return g.index_copy(-2, chosen, self.anchor(coordinates.index_select(-2, chosen)))

    def forward(self, coordinates, features):
        """coordinates [1,N,2] (shared grid), normalized features [B,N,3]."""
        g = self.embed_geometry(coordinates)
        f = self.physical(features)
        states, transports = [], []
        for size, assignment, encoder in zip(self.latent_sizes, self.assignment, self.encoder):
            p = sinkhorn_transport(assignment(g), self.iterations, self.temperature)
            transports.append(p)
            g = torch.matmul(p.transpose(-1, -2), g) * size
            f = encoder(torch.matmul(p.transpose(-1, -2), f) * size)
            states.append(f)
        f = self.decoder[-1](states[-1])
        for level in range(len(states) - 2, -1, -1):
            n = states[level].shape[-2]
            f = self.decoder[level](states[level] + torch.matmul(transports[level + 1], f) * n)
        return self.output(torch.matmul(transports[0], f) * features.shape[-2])