File size: 5,534 Bytes
db32cff | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """Independent implementation of MoNo-light, paper equations (1)--(6), (29)--(30).
No author repository code is included. See .trainer_work/darcy85/source_capture.md.
Identical Darcy coordinates share one geometry route across a batch. All transport
iterations remain differentiable, including both encoder and decoder projections.
"""
import copy
import math
import torch
from torch import nn
from torch.nn import functional as F
class ResidualProjector(nn.Module):
def __init__(self, inputs, width, outputs, dropout=0.0):
super().__init__()
self.lift = nn.Linear(inputs, width)
self.hidden = nn.ModuleList(nn.Linear(width, width) for _ in range(3))
self.drop = nn.Dropout(dropout)
self.readout = nn.Linear(width, outputs)
def forward(self, x):
x = F.gelu(self.lift(x))
for layer in self.hidden:
x = x + self.drop(F.gelu(layer(x)))
return self.readout(x)
def sinkhorn_transport(scores, iterations=8, temperature=1.0):
"""Uniform-marginal log Sinkhorn; finite iterations, never detached."""
z = scores / temperature
n, m = z.shape[-2:]
v = z.new_zeros((*z.shape[:-2], 1, m))
for _ in range(iterations):
u = -math.log(n) - torch.logsumexp(z + v, dim=-1, keepdim=True)
v = -math.log(m) - torch.logsumexp(z + u, dim=-2, keepdim=True)
return torch.exp(z + u + v)
class TransformerLayer(nn.Module):
def __init__(self, width, heads):
super().__init__()
self.heads = heads
self.norm_attention = nn.LayerNorm(width)
self.qkv = nn.Linear(width, width * 3)
self.attention_out = nn.Linear(width, width)
self.norm_ffn = nn.LayerNorm(width)
self.ffn = nn.Sequential(nn.Linear(width, 4 * width), nn.GELU(),
nn.Linear(4 * width, width))
def forward(self, x):
b, n, d = x.shape
q, k, v = self.qkv(self.norm_attention(x)).reshape(
b, n, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4).unbind(0)
a = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
x = x + self.attention_out(a.transpose(1, 2).reshape(b, n, d))
return x + self.ffn(self.norm_ffn(x))
def initialize_readout(module):
if isinstance(module, nn.Linear):
nn.init.trunc_normal_(module.weight, std=0.02)
if module.bias is not None:
nn.init.zeros_(module.bias)
elif isinstance(module, nn.LayerNorm):
nn.init.ones_(module.weight)
nn.init.zeros_(module.bias)
class MoNoLight(nn.Module):
def __init__(self, width=96, latent_sizes=(512, 256, 128, 64),
depths=(3, 1, 1, 1), heads=(4, 4, 8, 8),
sinkhorn_iterations=8, temperature=1.0,
anchor_grad_fraction=0.25, output_dropout=0.5):
super().__init__()
self.latent_sizes = tuple(latent_sizes)
self.iterations = sinkhorn_iterations
self.temperature = temperature
self.anchor_grad_fraction = anchor_grad_fraction
self.anchor = ResidualProjector(2, width, width)
self.frozen_anchor = copy.deepcopy(self.anchor).requires_grad_(False)
self.physical = ResidualProjector(3, width, width)
self.assignment = nn.ModuleList(
ResidualProjector(width, width, n) for n in latent_sizes)
self.encoder = nn.ModuleList(nn.Sequential(*[
TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads))
self.decoder = nn.ModuleList(nn.Sequential(*[
TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads))
self.output = ResidualProjector(width, width, 1, output_dropout)
self.encoder.apply(initialize_readout)
self.decoder.apply(initialize_readout)
self.output.apply(initialize_readout)
def embed_geometry(self, coordinates):
if not self.training:
return self.frozen_anchor(coordinates)
# Synchronize before the train forward, as documented in the existing spec.
with torch.no_grad():
for dst, src in zip(self.frozen_anchor.parameters(), self.anchor.parameters()):
dst.copy_(src)
n = coordinates.shape[-2]
chosen = torch.randperm(n, device=coordinates.device)[:math.ceil(n * self.anchor_grad_fraction)]
# Evaluate the frozen copy everywhere; replace chosen points by trainable outputs.
with torch.no_grad():
g = self.frozen_anchor(coordinates)
return g.index_copy(-2, chosen, self.anchor(coordinates.index_select(-2, chosen)))
def forward(self, coordinates, features):
"""coordinates [1,N,2] (shared grid), normalized features [B,N,3]."""
g = self.embed_geometry(coordinates)
f = self.physical(features)
states, transports = [], []
for size, assignment, encoder in zip(self.latent_sizes, self.assignment, self.encoder):
p = sinkhorn_transport(assignment(g), self.iterations, self.temperature)
transports.append(p)
g = torch.matmul(p.transpose(-1, -2), g) * size
f = encoder(torch.matmul(p.transpose(-1, -2), f) * size)
states.append(f)
f = self.decoder[-1](states[-1])
for level in range(len(states) - 2, -1, -1):
n = states[level].shape[-2]
f = self.decoder[level](states[level] + torch.matmul(transports[level + 1], f) * n)
return self.output(torch.matmul(transports[0], f) * features.shape[-2])
|