"""Independent implementation of SirenFNO, paper equations (5)-(7), (14). Dense complex kernels cover every rFFT bin. The RFF and initialization hyperparameters fill paper omissions using read-only official implementation observations. Paper-specified sine activations and 64-channel lifting/projection take precedence over conflicting repository choices. See reproduction_spec.md. """ import math import torch from torch import nn from torch.nn import functional as F class PointwiseMLP(nn.Module): def __init__(self, input_channels, hidden_channels, output_channels): super().__init__() self.layers = nn.Sequential( nn.Conv2d(input_channels, hidden_channels, 1), nn.GELU(), nn.Conv2d(hidden_channels, output_channels, 1), ) def forward(self, x): return self.layers(x) class KernelNetwork(nn.Module): """One real component of a complex mode-to-channel-matrix hypernetwork.""" def __init__(self, width, hidden, depth, features, sigma, omega, learnable_omega): super().__init__() self.frequencies = nn.Parameter(torch.randn(2, features // 2) * sigma) self.affine = nn.ModuleList() self.scales = nn.ParameterList() for index in range(depth): inputs = features if index == 0 else hidden layer = nn.Linear(inputs, hidden, bias=True) bound = 1.0 / inputs if index == 0 else math.sqrt(6.0 / inputs) / omega nn.init.uniform_(layer.weight, -bound, bound) nn.init.zeros_(layer.bias) self.affine.append(layer) self.scales.append(nn.Parameter(torch.rand(hidden) * omega, requires_grad=learnable_omega)) self.readout = nn.Linear(hidden, width * width, bias=False) bound = math.sqrt(6.0 / hidden) / omega nn.init.uniform_(self.readout.weight, -bound, bound) def forward(self, coordinates): phase = math.pi * (coordinates @ self.frequencies) hidden = torch.cat((phase.cos(), phase.sin()), dim=-1) for affine, scale in zip(self.affine, self.scales): # Paper eq. (7): sin(w W x + b); b starts at zero and is learned. hidden = torch.sin(F.linear(hidden, affine.weight) * scale + affine.bias) return self.readout(hidden) class FullSpectrumOperator(nn.Module): def __init__(self, width, **kernel_args): super().__init__() self.width = width self.real_kernel = KernelNetwork(width=width, **kernel_args) self.imag_kernel = KernelNetwork(width=width, **kernel_args) self.register_buffer("coordinates", torch.empty(0), persistent=False) self.coordinate_shape = None def mode_coordinates(self, height, half_width, x): shape = (height, half_width) if self.coordinate_shape != shape or self.coordinates.device != x.device: # Midpoints of normalized array-index coordinates, supplement from # official dense kernel. No frequency mask or zero-filled high bins. u = 2 * (torch.arange(height, device=x.device, dtype=x.dtype) + .5) / height - 1 v = 2 * (torch.arange(half_width, device=x.device, dtype=x.dtype) + .5) / half_width - 1 self.coordinates = torch.stack(torch.meshgrid(u, v, indexing="ij"), -1) self.coordinate_shape = shape return self.coordinates def forward(self, x): height, width = x.shape[-2:] half_width = width // 2 + 1 coordinates = self.mode_coordinates(height, half_width, x) real = self.real_kernel(coordinates).reshape(height, half_width, self.width, self.width) imag = self.imag_kernel(coordinates).reshape_as(real) kernel = torch.complex(real, imag) spectrum = torch.fft.rfft2(x, norm="backward") mixed = torch.einsum("bihw,hwio->bohw", spectrum, kernel) return torch.fft.irfft2(mixed.contiguous(), s=(height, width), norm="backward") class ResidualOperatorBlock(nn.Module): def __init__(self, width, expansion, activate, **kernel_args): super().__init__() self.spectral = FullSpectrumOperator(width, **kernel_args) self.channel_mlp = PointwiseMLP(width, width * expansion, width) self.activate = activate def forward(self, x): # Enhanced architecture eq. (14), not the vanilla-FNO ablation. result = x + self.channel_mlp(self.spectral(x)) return F.gelu(result) if self.activate else result class SirenFNO(nn.Module): def __init__(self, in_channels=1, out_channels=1, width=32, layers=4, lifting_channels=64, projection_channels=64, residual_expansion=4, siren_hidden=32, siren_layers=1, rff_features=16, rff_sigma=512.0, omega=30.0, learnable_omega=True, add_grid=True, truncation=False, factorization="dense", siren_activation="sin"): super().__init__() if truncation or factorization != "dense" or siren_activation != "sin": raise ValueError("This reproduction requires dense full-spectrum sine SirenFNO") if rff_features % 2 or siren_layers < 1: raise ValueError("RFF features must be even and SIREN depth positive") self.in_channels = in_channels self.add_grid = add_grid self.lifting = PointwiseMLP(in_channels + 2 * int(add_grid), lifting_channels, width) kernel_args = dict(hidden=siren_hidden, depth=siren_layers, features=rff_features, sigma=rff_sigma, omega=omega, learnable_omega=learnable_omega) self.blocks = nn.ModuleList([ ResidualOperatorBlock(width, residual_expansion, i < layers - 1, **kernel_args) for i in range(layers) ]) self.projection = PointwiseMLP(width, projection_channels, out_channels) def forward(self, x): if x.ndim != 4 or x.shape[1] != self.in_channels: raise ValueError(f"Expected NCHW permeability with {self.in_channels} channels: {x.shape}") if self.add_grid: batch, _, height, width = x.shape row = torch.linspace(0, 1, height, device=x.device, dtype=x.dtype) col = torch.linspace(0, 1, width, device=x.device, dtype=x.dtype) grid = torch.stack(torch.meshgrid(row, col, indexing="ij"), 0) x = torch.cat((x, grid.unsqueeze(0).expand(batch, -1, -1, -1)), 1) x = self.lifting(x) for block in self.blocks: x = block(x) return self.projection(x)