"""3D echocardiography masked autoencoder (transformer). Input: N x 1 x 256 x 64 x 64 beamspace volumes (radius x azimuth x elevation) in [0, 1]. """ import torch from torch import nn SHAPE = (256, 64, 64) # radius, azimuth, elevation PATCH = (16, 8, 8) GRID = (16, 8, 8) TOKENS = 1024 VOXELS = 1024 # voxels per patch def patchify(volume): (r, a, e), (pr, pa, pe) = GRID, PATCH volume = volume.reshape(-1, 1, r, pr, a, pa, e, pe) return volume.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(-1, TOKENS, VOXELS) def unpatchify(patches): (r, a, e), (pr, pa, pe) = GRID, PATCH patches = patches.reshape(-1, r, a, e, pr, pa, pe, 1) return patches.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(-1, 1, *SHAPE) class Block(nn.Module): def __init__(self, width, heads): super().__init__() self.norm1 = nn.LayerNorm(width) self.attention = nn.MultiheadAttention(width, heads, batch_first=True) self.norm2 = nn.LayerNorm(width) self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(), nn.Linear(width * 4, width)) def forward(self, x): normalized = self.norm1(x) x = x + self.attention(normalized, normalized, normalized, need_weights=False)[0] return x + self.mlp(self.norm2(x)) class EchoMAE3D(nn.Module): def __init__(self, width=384, depth=8, decoder_width=192, decoder_depth=4, mask_ratio=0.75): super().__init__() self.visible = int(TOKENS * (1 - mask_ratio)) self.embedding = nn.Linear(VOXELS, width) self.position = nn.Parameter(torch.zeros(1, TOKENS, width)) self.encoder = nn.Sequential(*[Block(width, 6) for _ in range(depth)]) self.encoder_norm = nn.LayerNorm(width) self.to_decoder = nn.Linear(width, decoder_width) self.mask_token = nn.Parameter(torch.zeros(1, 1, decoder_width)) self.decoder_position = nn.Parameter(torch.zeros(1, TOKENS, decoder_width)) self.decoder = nn.Sequential(*[Block(decoder_width, 6) for _ in range(decoder_depth)]) self.decoder_norm = nn.LayerNorm(decoder_width) self.prediction = nn.Linear(decoder_width, VOXELS) def encode(self, volume, keep): """Latent tokens of the patches listed in `keep` (N x visible indices).""" patches = patchify(volume) visible = patches.gather(1, keep[..., None].expand(-1, -1, VOXELS)) position = self.position.expand(len(patches), -1, -1) position = position.gather(1, keep[..., None].expand(-1, -1, position.shape[-1])) return self.encoder_norm(self.encoder(self.embedding(visible) + position)) def forward(self, volume, noise=None): """Mask random patches; return the reconstructed volume and the mask volume (1 = hidden).""" batch = volume.shape[0] if noise is None: noise = torch.rand(batch, TOKENS, device=volume.device) order = noise.argsort(dim=1) restore = order.argsort(dim=1) encoded = self.to_decoder(self.encode(volume, order[:, : self.visible])) missing = self.mask_token.expand(batch, TOKENS - self.visible, -1) tokens = torch.cat([encoded, missing], dim=1) tokens = tokens.gather(1, restore[..., None].expand(-1, -1, tokens.shape[-1])) predicted = self.prediction(self.decoder_norm(self.decoder(tokens + self.decoder_position))) mask = torch.ones(batch, TOKENS, device=volume.device) mask[:, : self.visible] = 0 mask = mask.gather(1, restore) return unpatchify(predicted), unpatchify(mask[..., None].expand(-1, -1, VOXELS))