Download model/MoNo.py from OneScience-Group/MoNo-light: direct link, hf CLI and curl.
- Browser
- Download file 5.53 kB
-
https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/model/MoNo.py
- Command line
-
hf download hf://OneScience-Group/MoNo-light/model/MoNo.py
-
curl -L -o MoNo.py https://huggingface.co/OneScience-Group/MoNo-light/resolve/main/model/MoNo.py
5.53 kB
| """Independent implementation of MoNo-light, paper equations (1)--(6), (29)--(30). | |
| No author repository code is included. See .trainer_work/darcy85/source_capture.md. | |
| Identical Darcy coordinates share one geometry route across a batch. All transport | |
| iterations remain differentiable, including both encoder and decoder projections. | |
| """ | |
| import copy | |
| import math | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class ResidualProjector(nn.Module): | |
| def __init__(self, inputs, width, outputs, dropout=0.0): | |
| super().__init__() | |
| self.lift = nn.Linear(inputs, width) | |
| self.hidden = nn.ModuleList(nn.Linear(width, width) for _ in range(3)) | |
| self.drop = nn.Dropout(dropout) | |
| self.readout = nn.Linear(width, outputs) | |
| def forward(self, x): | |
| x = F.gelu(self.lift(x)) | |
| for layer in self.hidden: | |
| x = x + self.drop(F.gelu(layer(x))) | |
| return self.readout(x) | |
| def sinkhorn_transport(scores, iterations=8, temperature=1.0): | |
| """Uniform-marginal log Sinkhorn; finite iterations, never detached.""" | |
| z = scores / temperature | |
| n, m = z.shape[-2:] | |
| v = z.new_zeros((*z.shape[:-2], 1, m)) | |
| for _ in range(iterations): | |
| u = -math.log(n) - torch.logsumexp(z + v, dim=-1, keepdim=True) | |
| v = -math.log(m) - torch.logsumexp(z + u, dim=-2, keepdim=True) | |
| return torch.exp(z + u + v) | |
| class TransformerLayer(nn.Module): | |
| def __init__(self, width, heads): | |
| super().__init__() | |
| self.heads = heads | |
| self.norm_attention = nn.LayerNorm(width) | |
| self.qkv = nn.Linear(width, width * 3) | |
| self.attention_out = nn.Linear(width, width) | |
| self.norm_ffn = nn.LayerNorm(width) | |
| self.ffn = nn.Sequential(nn.Linear(width, 4 * width), nn.GELU(), | |
| nn.Linear(4 * width, width)) | |
| def forward(self, x): | |
| b, n, d = x.shape | |
| q, k, v = self.qkv(self.norm_attention(x)).reshape( | |
| b, n, 3, self.heads, d // self.heads).permute(2, 0, 3, 1, 4).unbind(0) | |
| a = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0) | |
| x = x + self.attention_out(a.transpose(1, 2).reshape(b, n, d)) | |
| return x + self.ffn(self.norm_ffn(x)) | |
| def initialize_readout(module): | |
| if isinstance(module, nn.Linear): | |
| nn.init.trunc_normal_(module.weight, std=0.02) | |
| if module.bias is not None: | |
| nn.init.zeros_(module.bias) | |
| elif isinstance(module, nn.LayerNorm): | |
| nn.init.ones_(module.weight) | |
| nn.init.zeros_(module.bias) | |
| class MoNoLight(nn.Module): | |
| def __init__(self, width=96, latent_sizes=(512, 256, 128, 64), | |
| depths=(3, 1, 1, 1), heads=(4, 4, 8, 8), | |
| sinkhorn_iterations=8, temperature=1.0, | |
| anchor_grad_fraction=0.25, output_dropout=0.5): | |
| super().__init__() | |
| self.latent_sizes = tuple(latent_sizes) | |
| self.iterations = sinkhorn_iterations | |
| self.temperature = temperature | |
| self.anchor_grad_fraction = anchor_grad_fraction | |
| self.anchor = ResidualProjector(2, width, width) | |
| self.frozen_anchor = copy.deepcopy(self.anchor).requires_grad_(False) | |
| self.physical = ResidualProjector(3, width, width) | |
| self.assignment = nn.ModuleList( | |
| ResidualProjector(width, width, n) for n in latent_sizes) | |
| self.encoder = nn.ModuleList(nn.Sequential(*[ | |
| TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads)) | |
| self.decoder = nn.ModuleList(nn.Sequential(*[ | |
| TransformerLayer(width, h) for _ in range(d)]) for d, h in zip(depths, heads)) | |
| self.output = ResidualProjector(width, width, 1, output_dropout) | |
| self.encoder.apply(initialize_readout) | |
| self.decoder.apply(initialize_readout) | |
| self.output.apply(initialize_readout) | |
| def embed_geometry(self, coordinates): | |
| if not self.training: | |
| return self.frozen_anchor(coordinates) | |
| # Synchronize before the train forward, as documented in the existing spec. | |
| with torch.no_grad(): | |
| for dst, src in zip(self.frozen_anchor.parameters(), self.anchor.parameters()): | |
| dst.copy_(src) | |
| n = coordinates.shape[-2] | |
| chosen = torch.randperm(n, device=coordinates.device)[:math.ceil(n * self.anchor_grad_fraction)] | |
| # Evaluate the frozen copy everywhere; replace chosen points by trainable outputs. | |
| with torch.no_grad(): | |
| g = self.frozen_anchor(coordinates) | |
| return g.index_copy(-2, chosen, self.anchor(coordinates.index_select(-2, chosen))) | |
| def forward(self, coordinates, features): | |
| """coordinates [1,N,2] (shared grid), normalized features [B,N,3].""" | |
| g = self.embed_geometry(coordinates) | |
| f = self.physical(features) | |
| states, transports = [], [] | |
| for size, assignment, encoder in zip(self.latent_sizes, self.assignment, self.encoder): | |
| p = sinkhorn_transport(assignment(g), self.iterations, self.temperature) | |
| transports.append(p) | |
| g = torch.matmul(p.transpose(-1, -2), g) * size | |
| f = encoder(torch.matmul(p.transpose(-1, -2), f) * size) | |
| states.append(f) | |
| f = self.decoder[-1](states[-1]) | |
| for level in range(len(states) - 2, -1, -1): | |
| n = states[level].shape[-2] | |
| f = self.decoder[level](states[level] + torch.matmul(transports[level + 1], f) * n) | |
| return self.output(torch.matmul(transports[0], f) * features.shape[-2]) | |