File size: 6,597 Bytes
10ab127 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | """Independent implementation of SirenFNO, paper equations (5)-(7), (14).
Dense complex kernels cover every rFFT bin. The RFF and initialization
hyperparameters fill paper omissions using read-only official implementation
observations. Paper-specified sine activations and 64-channel lifting/projection
take precedence over conflicting repository choices. See reproduction_spec.md.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
class PointwiseMLP(nn.Module):
def __init__(self, input_channels, hidden_channels, output_channels):
super().__init__()
self.layers = nn.Sequential(
nn.Conv2d(input_channels, hidden_channels, 1),
nn.GELU(),
nn.Conv2d(hidden_channels, output_channels, 1),
)
def forward(self, x):
return self.layers(x)
class KernelNetwork(nn.Module):
"""One real component of a complex mode-to-channel-matrix hypernetwork."""
def __init__(self, width, hidden, depth, features, sigma, omega, learnable_omega):
super().__init__()
self.frequencies = nn.Parameter(torch.randn(2, features // 2) * sigma)
self.affine = nn.ModuleList()
self.scales = nn.ParameterList()
for index in range(depth):
inputs = features if index == 0 else hidden
layer = nn.Linear(inputs, hidden, bias=True)
bound = 1.0 / inputs if index == 0 else math.sqrt(6.0 / inputs) / omega
nn.init.uniform_(layer.weight, -bound, bound)
nn.init.zeros_(layer.bias)
self.affine.append(layer)
self.scales.append(nn.Parameter(torch.rand(hidden) * omega,
requires_grad=learnable_omega))
self.readout = nn.Linear(hidden, width * width, bias=False)
bound = math.sqrt(6.0 / hidden) / omega
nn.init.uniform_(self.readout.weight, -bound, bound)
def forward(self, coordinates):
phase = math.pi * (coordinates @ self.frequencies)
hidden = torch.cat((phase.cos(), phase.sin()), dim=-1)
for affine, scale in zip(self.affine, self.scales):
# Paper eq. (7): sin(w W x + b); b starts at zero and is learned.
hidden = torch.sin(F.linear(hidden, affine.weight) * scale + affine.bias)
return self.readout(hidden)
class FullSpectrumOperator(nn.Module):
def __init__(self, width, **kernel_args):
super().__init__()
self.width = width
self.real_kernel = KernelNetwork(width=width, **kernel_args)
self.imag_kernel = KernelNetwork(width=width, **kernel_args)
self.register_buffer("coordinates", torch.empty(0), persistent=False)
self.coordinate_shape = None
def mode_coordinates(self, height, half_width, x):
shape = (height, half_width)
if self.coordinate_shape != shape or self.coordinates.device != x.device:
# Midpoints of normalized array-index coordinates, supplement from
# official dense kernel. No frequency mask or zero-filled high bins.
u = 2 * (torch.arange(height, device=x.device, dtype=x.dtype) + .5) / height - 1
v = 2 * (torch.arange(half_width, device=x.device, dtype=x.dtype) + .5) / half_width - 1
self.coordinates = torch.stack(torch.meshgrid(u, v, indexing="ij"), -1)
self.coordinate_shape = shape
return self.coordinates
def forward(self, x):
height, width = x.shape[-2:]
half_width = width // 2 + 1
coordinates = self.mode_coordinates(height, half_width, x)
real = self.real_kernel(coordinates).reshape(height, half_width, self.width, self.width)
imag = self.imag_kernel(coordinates).reshape_as(real)
kernel = torch.complex(real, imag)
spectrum = torch.fft.rfft2(x, norm="backward")
mixed = torch.einsum("bihw,hwio->bohw", spectrum, kernel)
return torch.fft.irfft2(mixed.contiguous(), s=(height, width), norm="backward")
class ResidualOperatorBlock(nn.Module):
def __init__(self, width, expansion, activate, **kernel_args):
super().__init__()
self.spectral = FullSpectrumOperator(width, **kernel_args)
self.channel_mlp = PointwiseMLP(width, width * expansion, width)
self.activate = activate
def forward(self, x):
# Enhanced architecture eq. (14), not the vanilla-FNO ablation.
result = x + self.channel_mlp(self.spectral(x))
return F.gelu(result) if self.activate else result
class SirenFNO(nn.Module):
def __init__(self, in_channels=1, out_channels=1, width=32, layers=4,
lifting_channels=64, projection_channels=64,
residual_expansion=4, siren_hidden=32, siren_layers=1,
rff_features=16, rff_sigma=512.0, omega=30.0,
learnable_omega=True, add_grid=True, truncation=False,
factorization="dense", siren_activation="sin"):
super().__init__()
if truncation or factorization != "dense" or siren_activation != "sin":
raise ValueError("This reproduction requires dense full-spectrum sine SirenFNO")
if rff_features % 2 or siren_layers < 1:
raise ValueError("RFF features must be even and SIREN depth positive")
self.in_channels = in_channels
self.add_grid = add_grid
self.lifting = PointwiseMLP(in_channels + 2 * int(add_grid), lifting_channels, width)
kernel_args = dict(hidden=siren_hidden, depth=siren_layers, features=rff_features,
sigma=rff_sigma, omega=omega, learnable_omega=learnable_omega)
self.blocks = nn.ModuleList([
ResidualOperatorBlock(width, residual_expansion, i < layers - 1, **kernel_args)
for i in range(layers)
])
self.projection = PointwiseMLP(width, projection_channels, out_channels)
def forward(self, x):
if x.ndim != 4 or x.shape[1] != self.in_channels:
raise ValueError(f"Expected NCHW permeability with {self.in_channels} channels: {x.shape}")
if self.add_grid:
batch, _, height, width = x.shape
row = torch.linspace(0, 1, height, device=x.device, dtype=x.dtype)
col = torch.linspace(0, 1, width, device=x.device, dtype=x.dtype)
grid = torch.stack(torch.meshgrid(row, col, indexing="ij"), 0)
x = torch.cat((x, grid.unsqueeze(0).expand(batch, -1, -1, -1)), 1)
x = self.lifting(x)
for block in self.blocks:
x = block(x)
return self.projection(x)
|