SirenFNO / model /SirenFNO.py
yushuang88's picture
Upload folder using huggingface_hub
10ab127 verified
Raw History Blame Contribute Delete
6.6 kB
"""Independent implementation of SirenFNO, paper equations (5)-(7), (14).
Dense complex kernels cover every rFFT bin. The RFF and initialization
hyperparameters fill paper omissions using read-only official implementation
observations. Paper-specified sine activations and 64-channel lifting/projection
take precedence over conflicting repository choices. See reproduction_spec.md.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
class PointwiseMLP(nn.Module):
def __init__(self, input_channels, hidden_channels, output_channels):
super().__init__()
self.layers = nn.Sequential(
nn.Conv2d(input_channels, hidden_channels, 1),
nn.GELU(),
nn.Conv2d(hidden_channels, output_channels, 1),
)
def forward(self, x):
return self.layers(x)
class KernelNetwork(nn.Module):
"""One real component of a complex mode-to-channel-matrix hypernetwork."""
def __init__(self, width, hidden, depth, features, sigma, omega, learnable_omega):
super().__init__()
self.frequencies = nn.Parameter(torch.randn(2, features // 2) * sigma)
self.affine = nn.ModuleList()
self.scales = nn.ParameterList()
for index in range(depth):
inputs = features if index == 0 else hidden
layer = nn.Linear(inputs, hidden, bias=True)
bound = 1.0 / inputs if index == 0 else math.sqrt(6.0 / inputs) / omega
nn.init.uniform_(layer.weight, -bound, bound)
nn.init.zeros_(layer.bias)
self.affine.append(layer)
self.scales.append(nn.Parameter(torch.rand(hidden) * omega,
requires_grad=learnable_omega))
self.readout = nn.Linear(hidden, width * width, bias=False)
bound = math.sqrt(6.0 / hidden) / omega
nn.init.uniform_(self.readout.weight, -bound, bound)
def forward(self, coordinates):
phase = math.pi * (coordinates @ self.frequencies)
hidden = torch.cat((phase.cos(), phase.sin()), dim=-1)
for affine, scale in zip(self.affine, self.scales):
# Paper eq. (7): sin(w W x + b); b starts at zero and is learned.
hidden = torch.sin(F.linear(hidden, affine.weight) * scale + affine.bias)
return self.readout(hidden)
class FullSpectrumOperator(nn.Module):
def __init__(self, width, **kernel_args):
super().__init__()
self.width = width
self.real_kernel = KernelNetwork(width=width, **kernel_args)
self.imag_kernel = KernelNetwork(width=width, **kernel_args)
self.register_buffer("coordinates", torch.empty(0), persistent=False)
self.coordinate_shape = None
def mode_coordinates(self, height, half_width, x):
shape = (height, half_width)
if self.coordinate_shape != shape or self.coordinates.device != x.device:
# Midpoints of normalized array-index coordinates, supplement from
# official dense kernel. No frequency mask or zero-filled high bins.
u = 2 * (torch.arange(height, device=x.device, dtype=x.dtype) + .5) / height - 1
v = 2 * (torch.arange(half_width, device=x.device, dtype=x.dtype) + .5) / half_width - 1
self.coordinates = torch.stack(torch.meshgrid(u, v, indexing="ij"), -1)
self.coordinate_shape = shape
return self.coordinates
def forward(self, x):
height, width = x.shape[-2:]
half_width = width // 2 + 1
coordinates = self.mode_coordinates(height, half_width, x)
real = self.real_kernel(coordinates).reshape(height, half_width, self.width, self.width)
imag = self.imag_kernel(coordinates).reshape_as(real)
kernel = torch.complex(real, imag)
spectrum = torch.fft.rfft2(x, norm="backward")
mixed = torch.einsum("bihw,hwio->bohw", spectrum, kernel)
return torch.fft.irfft2(mixed.contiguous(), s=(height, width), norm="backward")
class ResidualOperatorBlock(nn.Module):
def __init__(self, width, expansion, activate, **kernel_args):
super().__init__()
self.spectral = FullSpectrumOperator(width, **kernel_args)
self.channel_mlp = PointwiseMLP(width, width * expansion, width)
self.activate = activate
def forward(self, x):
# Enhanced architecture eq. (14), not the vanilla-FNO ablation.
result = x + self.channel_mlp(self.spectral(x))
return F.gelu(result) if self.activate else result
class SirenFNO(nn.Module):
def __init__(self, in_channels=1, out_channels=1, width=32, layers=4,
lifting_channels=64, projection_channels=64,
residual_expansion=4, siren_hidden=32, siren_layers=1,
rff_features=16, rff_sigma=512.0, omega=30.0,
learnable_omega=True, add_grid=True, truncation=False,
factorization="dense", siren_activation="sin"):
super().__init__()
if truncation or factorization != "dense" or siren_activation != "sin":
raise ValueError("This reproduction requires dense full-spectrum sine SirenFNO")
if rff_features % 2 or siren_layers < 1:
raise ValueError("RFF features must be even and SIREN depth positive")
self.in_channels = in_channels
self.add_grid = add_grid
self.lifting = PointwiseMLP(in_channels + 2 * int(add_grid), lifting_channels, width)
kernel_args = dict(hidden=siren_hidden, depth=siren_layers, features=rff_features,
sigma=rff_sigma, omega=omega, learnable_omega=learnable_omega)
self.blocks = nn.ModuleList([
ResidualOperatorBlock(width, residual_expansion, i < layers - 1, **kernel_args)
for i in range(layers)
])
self.projection = PointwiseMLP(width, projection_channels, out_channels)
def forward(self, x):
if x.ndim != 4 or x.shape[1] != self.in_channels:
raise ValueError(f"Expected NCHW permeability with {self.in_channels} channels: {x.shape}")
if self.add_grid:
batch, _, height, width = x.shape
row = torch.linspace(0, 1, height, device=x.device, dtype=x.dtype)
col = torch.linspace(0, 1, width, device=x.device, dtype=x.dtype)
grid = torch.stack(torch.meshgrid(row, col, indexing="ij"), 0)
x = torch.cat((x, grid.unsqueeze(0).expand(batch, -1, -1, -1)), 1)
x = self.lifting(x)
for block in self.blocks:
x = block(x)
return self.projection(x)