Download model.py from Ahus-AIM/echo-mae-3d: direct link, hf CLI and curl.
- Browser
- Download file 3.59 kB
-
https://huggingface.co/Ahus-AIM/echo-mae-3d/resolve/main/model.py
- Command line
-
hf download hf://Ahus-AIM/echo-mae-3d/model.py
-
curl -L -o model.py https://huggingface.co/Ahus-AIM/echo-mae-3d/resolve/main/model.py
3.59 kB
| """3D echocardiography masked autoencoder (transformer). | |
| Input: N x 1 x 256 x 64 x 64 beamspace volumes (radius x azimuth x elevation) in [0, 1]. | |
| """ | |
| import torch | |
| from torch import nn | |
| SHAPE = (256, 64, 64) # radius, azimuth, elevation | |
| PATCH = (16, 8, 8) | |
| GRID = (16, 8, 8) | |
| TOKENS = 1024 | |
| VOXELS = 1024 # voxels per patch | |
| def patchify(volume): | |
| (r, a, e), (pr, pa, pe) = GRID, PATCH | |
| volume = volume.reshape(-1, 1, r, pr, a, pa, e, pe) | |
| return volume.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(-1, TOKENS, VOXELS) | |
| def unpatchify(patches): | |
| (r, a, e), (pr, pa, pe) = GRID, PATCH | |
| patches = patches.reshape(-1, r, a, e, pr, pa, pe, 1) | |
| return patches.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(-1, 1, *SHAPE) | |
| class Block(nn.Module): | |
| def __init__(self, width, heads): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(width) | |
| self.attention = nn.MultiheadAttention(width, heads, batch_first=True) | |
| self.norm2 = nn.LayerNorm(width) | |
| self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(), nn.Linear(width * 4, width)) | |
| def forward(self, x): | |
| normalized = self.norm1(x) | |
| x = x + self.attention(normalized, normalized, normalized, need_weights=False)[0] | |
| return x + self.mlp(self.norm2(x)) | |
| class EchoMAE3D(nn.Module): | |
| def __init__(self, width=384, depth=8, decoder_width=192, decoder_depth=4, mask_ratio=0.75): | |
| super().__init__() | |
| self.visible = int(TOKENS * (1 - mask_ratio)) | |
| self.embedding = nn.Linear(VOXELS, width) | |
| self.position = nn.Parameter(torch.zeros(1, TOKENS, width)) | |
| self.encoder = nn.Sequential(*[Block(width, 6) for _ in range(depth)]) | |
| self.encoder_norm = nn.LayerNorm(width) | |
| self.to_decoder = nn.Linear(width, decoder_width) | |
| self.mask_token = nn.Parameter(torch.zeros(1, 1, decoder_width)) | |
| self.decoder_position = nn.Parameter(torch.zeros(1, TOKENS, decoder_width)) | |
| self.decoder = nn.Sequential(*[Block(decoder_width, 6) for _ in range(decoder_depth)]) | |
| self.decoder_norm = nn.LayerNorm(decoder_width) | |
| self.prediction = nn.Linear(decoder_width, VOXELS) | |
| def encode(self, volume, keep): | |
| """Latent tokens of the patches listed in `keep` (N x visible indices).""" | |
| patches = patchify(volume) | |
| visible = patches.gather(1, keep[..., None].expand(-1, -1, VOXELS)) | |
| position = self.position.expand(len(patches), -1, -1) | |
| position = position.gather(1, keep[..., None].expand(-1, -1, position.shape[-1])) | |
| return self.encoder_norm(self.encoder(self.embedding(visible) + position)) | |
| def forward(self, volume, noise=None): | |
| """Mask random patches; return the reconstructed volume and the mask volume (1 = hidden).""" | |
| batch = volume.shape[0] | |
| if noise is None: | |
| noise = torch.rand(batch, TOKENS, device=volume.device) | |
| order = noise.argsort(dim=1) | |
| restore = order.argsort(dim=1) | |
| encoded = self.to_decoder(self.encode(volume, order[:, : self.visible])) | |
| missing = self.mask_token.expand(batch, TOKENS - self.visible, -1) | |
| tokens = torch.cat([encoded, missing], dim=1) | |
| tokens = tokens.gather(1, restore[..., None].expand(-1, -1, tokens.shape[-1])) | |
| predicted = self.prediction(self.decoder_norm(self.decoder(tokens + self.decoder_position))) | |
| mask = torch.ones(batch, TOKENS, device=volume.device) | |
| mask[:, : self.visible] = 0 | |
| mask = mask.gather(1, restore) | |
| return unpatchify(predicted), unpatchify(mask[..., None].expand(-1, -1, VOXELS)) | |