File size: 6,597 Bytes
10ab127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
"""Independent implementation of SirenFNO, paper equations (5)-(7), (14).

Dense complex kernels cover every rFFT bin. The RFF and initialization
hyperparameters fill paper omissions using read-only official implementation
observations. Paper-specified sine activations and 64-channel lifting/projection
take precedence over conflicting repository choices. See reproduction_spec.md.
"""

import math

import torch
from torch import nn
from torch.nn import functional as F


class PointwiseMLP(nn.Module):
    def __init__(self, input_channels, hidden_channels, output_channels):
        super().__init__()
        self.layers = nn.Sequential(
            nn.Conv2d(input_channels, hidden_channels, 1),
            nn.GELU(),
            nn.Conv2d(hidden_channels, output_channels, 1),
        )

    def forward(self, x):
        return self.layers(x)


class KernelNetwork(nn.Module):
    """One real component of a complex mode-to-channel-matrix hypernetwork."""

    def __init__(self, width, hidden, depth, features, sigma, omega, learnable_omega):
        super().__init__()
        self.frequencies = nn.Parameter(torch.randn(2, features // 2) * sigma)
        self.affine = nn.ModuleList()
        self.scales = nn.ParameterList()
        for index in range(depth):
            inputs = features if index == 0 else hidden
            layer = nn.Linear(inputs, hidden, bias=True)
            bound = 1.0 / inputs if index == 0 else math.sqrt(6.0 / inputs) / omega
            nn.init.uniform_(layer.weight, -bound, bound)
            nn.init.zeros_(layer.bias)
            self.affine.append(layer)
            self.scales.append(nn.Parameter(torch.rand(hidden) * omega,
                                            requires_grad=learnable_omega))
        self.readout = nn.Linear(hidden, width * width, bias=False)
        bound = math.sqrt(6.0 / hidden) / omega
        nn.init.uniform_(self.readout.weight, -bound, bound)

    def forward(self, coordinates):
        phase = math.pi * (coordinates @ self.frequencies)
        hidden = torch.cat((phase.cos(), phase.sin()), dim=-1)
        for affine, scale in zip(self.affine, self.scales):
            # Paper eq. (7): sin(w W x + b); b starts at zero and is learned.
            hidden = torch.sin(F.linear(hidden, affine.weight) * scale + affine.bias)
        return self.readout(hidden)


class FullSpectrumOperator(nn.Module):
    def __init__(self, width, **kernel_args):
        super().__init__()
        self.width = width
        self.real_kernel = KernelNetwork(width=width, **kernel_args)
        self.imag_kernel = KernelNetwork(width=width, **kernel_args)
        self.register_buffer("coordinates", torch.empty(0), persistent=False)
        self.coordinate_shape = None

    def mode_coordinates(self, height, half_width, x):
        shape = (height, half_width)
        if self.coordinate_shape != shape or self.coordinates.device != x.device:
            # Midpoints of normalized array-index coordinates, supplement from
            # official dense kernel. No frequency mask or zero-filled high bins.
            u = 2 * (torch.arange(height, device=x.device, dtype=x.dtype) + .5) / height - 1
            v = 2 * (torch.arange(half_width, device=x.device, dtype=x.dtype) + .5) / half_width - 1
            self.coordinates = torch.stack(torch.meshgrid(u, v, indexing="ij"), -1)
            self.coordinate_shape = shape
        return self.coordinates

    def forward(self, x):
        height, width = x.shape[-2:]
        half_width = width // 2 + 1
        coordinates = self.mode_coordinates(height, half_width, x)
        real = self.real_kernel(coordinates).reshape(height, half_width, self.width, self.width)
        imag = self.imag_kernel(coordinates).reshape_as(real)
        kernel = torch.complex(real, imag)
        spectrum = torch.fft.rfft2(x, norm="backward")
        mixed = torch.einsum("bihw,hwio->bohw", spectrum, kernel)
        return torch.fft.irfft2(mixed.contiguous(), s=(height, width), norm="backward")


class ResidualOperatorBlock(nn.Module):
    def __init__(self, width, expansion, activate, **kernel_args):
        super().__init__()
        self.spectral = FullSpectrumOperator(width, **kernel_args)
        self.channel_mlp = PointwiseMLP(width, width * expansion, width)
        self.activate = activate

    def forward(self, x):
        # Enhanced architecture eq. (14), not the vanilla-FNO ablation.
        result = x + self.channel_mlp(self.spectral(x))
        return F.gelu(result) if self.activate else result


class SirenFNO(nn.Module):
    def __init__(self, in_channels=1, out_channels=1, width=32, layers=4,
                 lifting_channels=64, projection_channels=64,
                 residual_expansion=4, siren_hidden=32, siren_layers=1,
                 rff_features=16, rff_sigma=512.0, omega=30.0,
                 learnable_omega=True, add_grid=True, truncation=False,
                 factorization="dense", siren_activation="sin"):
        super().__init__()
        if truncation or factorization != "dense" or siren_activation != "sin":
            raise ValueError("This reproduction requires dense full-spectrum sine SirenFNO")
        if rff_features % 2 or siren_layers < 1:
            raise ValueError("RFF features must be even and SIREN depth positive")
        self.in_channels = in_channels
        self.add_grid = add_grid
        self.lifting = PointwiseMLP(in_channels + 2 * int(add_grid), lifting_channels, width)
        kernel_args = dict(hidden=siren_hidden, depth=siren_layers, features=rff_features,
                           sigma=rff_sigma, omega=omega, learnable_omega=learnable_omega)
        self.blocks = nn.ModuleList([
            ResidualOperatorBlock(width, residual_expansion, i < layers - 1, **kernel_args)
            for i in range(layers)
        ])
        self.projection = PointwiseMLP(width, projection_channels, out_channels)

    def forward(self, x):
        if x.ndim != 4 or x.shape[1] != self.in_channels:
            raise ValueError(f"Expected NCHW permeability with {self.in_channels} channels: {x.shape}")
        if self.add_grid:
            batch, _, height, width = x.shape
            row = torch.linspace(0, 1, height, device=x.device, dtype=x.dtype)
            col = torch.linspace(0, 1, width, device=x.device, dtype=x.dtype)
            grid = torch.stack(torch.meshgrid(row, col, indexing="ij"), 0)
            x = torch.cat((x, grid.unsqueeze(0).expand(batch, -1, -1, -1)), 1)
        x = self.lifting(x)
        for block in self.blocks:
            x = block(x)
        return self.projection(x)