PyTorch
echocardiography
ultrasound
masked-autoencoder
3d
echo-mae-3d / model.py
eliasstenhede's picture
Model weights, code, sample and model card
c9e97e7 verified
Raw History Blame Contribute Delete
3.59 kB
"""3D echocardiography masked autoencoder (transformer).
Input: N x 1 x 256 x 64 x 64 beamspace volumes (radius x azimuth x elevation) in [0, 1].
"""
import torch
from torch import nn
SHAPE = (256, 64, 64) # radius, azimuth, elevation
PATCH = (16, 8, 8)
GRID = (16, 8, 8)
TOKENS = 1024
VOXELS = 1024 # voxels per patch
def patchify(volume):
(r, a, e), (pr, pa, pe) = GRID, PATCH
volume = volume.reshape(-1, 1, r, pr, a, pa, e, pe)
return volume.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(-1, TOKENS, VOXELS)
def unpatchify(patches):
(r, a, e), (pr, pa, pe) = GRID, PATCH
patches = patches.reshape(-1, r, a, e, pr, pa, pe, 1)
return patches.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(-1, 1, *SHAPE)
class Block(nn.Module):
def __init__(self, width, heads):
super().__init__()
self.norm1 = nn.LayerNorm(width)
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
self.norm2 = nn.LayerNorm(width)
self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(), nn.Linear(width * 4, width))
def forward(self, x):
normalized = self.norm1(x)
x = x + self.attention(normalized, normalized, normalized, need_weights=False)[0]
return x + self.mlp(self.norm2(x))
class EchoMAE3D(nn.Module):
def __init__(self, width=384, depth=8, decoder_width=192, decoder_depth=4, mask_ratio=0.75):
super().__init__()
self.visible = int(TOKENS * (1 - mask_ratio))
self.embedding = nn.Linear(VOXELS, width)
self.position = nn.Parameter(torch.zeros(1, TOKENS, width))
self.encoder = nn.Sequential(*[Block(width, 6) for _ in range(depth)])
self.encoder_norm = nn.LayerNorm(width)
self.to_decoder = nn.Linear(width, decoder_width)
self.mask_token = nn.Parameter(torch.zeros(1, 1, decoder_width))
self.decoder_position = nn.Parameter(torch.zeros(1, TOKENS, decoder_width))
self.decoder = nn.Sequential(*[Block(decoder_width, 6) for _ in range(decoder_depth)])
self.decoder_norm = nn.LayerNorm(decoder_width)
self.prediction = nn.Linear(decoder_width, VOXELS)
def encode(self, volume, keep):
"""Latent tokens of the patches listed in `keep` (N x visible indices)."""
patches = patchify(volume)
visible = patches.gather(1, keep[..., None].expand(-1, -1, VOXELS))
position = self.position.expand(len(patches), -1, -1)
position = position.gather(1, keep[..., None].expand(-1, -1, position.shape[-1]))
return self.encoder_norm(self.encoder(self.embedding(visible) + position))
def forward(self, volume, noise=None):
"""Mask random patches; return the reconstructed volume and the mask volume (1 = hidden)."""
batch = volume.shape[0]
if noise is None:
noise = torch.rand(batch, TOKENS, device=volume.device)
order = noise.argsort(dim=1)
restore = order.argsort(dim=1)
encoded = self.to_decoder(self.encode(volume, order[:, : self.visible]))
missing = self.mask_token.expand(batch, TOKENS - self.visible, -1)
tokens = torch.cat([encoded, missing], dim=1)
tokens = tokens.gather(1, restore[..., None].expand(-1, -1, tokens.shape[-1]))
predicted = self.prediction(self.decoder_norm(self.decoder(tokens + self.decoder_position)))
mask = torch.ones(batch, TOKENS, device=volume.device)
mask[:, : self.visible] = 0
mask = mask.gather(1, restore)
return unpatchify(predicted), unpatchify(mask[..., None].expand(-1, -1, VOXELS))