"""2D echocardiography masked autoencoder (convolutional, U-Net decoder). Input and output: N x 1 x 256 x 128 beamspace frames (depth x lateral) in [0, 1]. The architecture is a PyTorch adaptation of the MAE in lambertae/drifting (MIT license). """ import torch from torch import nn from torch.nn import functional as F def norm(channels): return nn.BatchNorm2d(channels, eps=0.01) def conv(inputs, outputs, kernel=3, stride=1): return nn.Conv2d(inputs, outputs, kernel, stride, kernel // 2, bias=False) def conv_norm_relu(inputs, outputs): return nn.Sequential(conv(inputs, outputs), norm(outputs), nn.ReLU()) class BasicBlock(nn.Module): def __init__(self, inputs, outputs, stride=1): super().__init__() self.body = nn.Sequential( conv(inputs, outputs, stride=stride), norm(outputs), nn.ReLU(), conv(outputs, outputs), norm(outputs) ) same = inputs == outputs and stride == 1 self.skip = nn.Identity() if same else nn.Sequential(conv(inputs, outputs, 1, stride), norm(outputs)) def forward(self, x): return F.relu(self.body(x) + self.skip(x)) class Up(nn.Module): def __init__(self, inputs, skip, outputs): super().__init__() self.body = nn.Sequential( norm(inputs + skip), conv_norm_relu(inputs + skip, outputs), conv_norm_relu(outputs, outputs) ) def forward(self, x, skip): x = F.interpolate(x, size=skip.shape[-2:], mode="bilinear", align_corners=False) return self.body(torch.cat((x, skip), 1)) class EchoMAE2D(nn.Module): def __init__(self, width=320, layers=(3, 4, 6, 3)): super().__init__() self.stem = conv_norm_relu(64, width) # one channel after 8 x 8 pixel-unshuffle self.stages = nn.ModuleList() self.norms = nn.ModuleList() inputs = width for index, blocks in enumerate(layers): outputs = width * 2**index stage = [BasicBlock(inputs, outputs, 2 if index else 1)] stage += [BasicBlock(outputs, outputs) for _ in range(blocks - 1)] self.stages.append(nn.Sequential(*stage)) self.norms.append(norm(outputs)) inputs = outputs self.bridge = conv_norm_relu(8 * width, 8 * width) self.ups = nn.ModuleList( [ Up(8 * width, 4 * width, 4 * width), Up(4 * width, 2 * width, 2 * width), Up(2 * width, width, width), Up(width, width, width), ] ) self.head = nn.Conv2d(width, 64, 1) def encode(self, x): """Feature maps at 1/8, 1/8, 1/16, 1/32 and 1/64 of the input resolution.""" x = self.stem(F.pixel_unshuffle(x, 8)) features = [x] for stage, layer_norm in zip(self.stages, self.norms): x = layer_norm(stage(x)) features.append(x) return features def forward(self, x): """Reconstruct the full frame from a (masked) frame.""" features = self.encode(x) y = self.bridge(features[-1]) for up, skip in zip(self.ups, reversed(features[:-1])): y = up(y, skip) return F.pixel_shuffle(self.head(y), 8)