PyTorch
echocardiography
ultrasound
masked-autoencoder
3d
File size: 3,590 Bytes
c9e97e7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
"""3D echocardiography masked autoencoder (transformer).

Input: N x 1 x 256 x 64 x 64 beamspace volumes (radius x azimuth x elevation) in [0, 1].
"""

import torch
from torch import nn

SHAPE = (256, 64, 64)  # radius, azimuth, elevation
PATCH = (16, 8, 8)
GRID = (16, 8, 8)
TOKENS = 1024
VOXELS = 1024  # voxels per patch


def patchify(volume):
    (r, a, e), (pr, pa, pe) = GRID, PATCH
    volume = volume.reshape(-1, 1, r, pr, a, pa, e, pe)
    return volume.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(-1, TOKENS, VOXELS)


def unpatchify(patches):
    (r, a, e), (pr, pa, pe) = GRID, PATCH
    patches = patches.reshape(-1, r, a, e, pr, pa, pe, 1)
    return patches.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(-1, 1, *SHAPE)


class Block(nn.Module):
    def __init__(self, width, heads):
        super().__init__()
        self.norm1 = nn.LayerNorm(width)
        self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
        self.norm2 = nn.LayerNorm(width)
        self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(), nn.Linear(width * 4, width))

    def forward(self, x):
        normalized = self.norm1(x)
        x = x + self.attention(normalized, normalized, normalized, need_weights=False)[0]
        return x + self.mlp(self.norm2(x))


class EchoMAE3D(nn.Module):
    def __init__(self, width=384, depth=8, decoder_width=192, decoder_depth=4, mask_ratio=0.75):
        super().__init__()
        self.visible = int(TOKENS * (1 - mask_ratio))
        self.embedding = nn.Linear(VOXELS, width)
        self.position = nn.Parameter(torch.zeros(1, TOKENS, width))
        self.encoder = nn.Sequential(*[Block(width, 6) for _ in range(depth)])
        self.encoder_norm = nn.LayerNorm(width)
        self.to_decoder = nn.Linear(width, decoder_width)
        self.mask_token = nn.Parameter(torch.zeros(1, 1, decoder_width))
        self.decoder_position = nn.Parameter(torch.zeros(1, TOKENS, decoder_width))
        self.decoder = nn.Sequential(*[Block(decoder_width, 6) for _ in range(decoder_depth)])
        self.decoder_norm = nn.LayerNorm(decoder_width)
        self.prediction = nn.Linear(decoder_width, VOXELS)

    def encode(self, volume, keep):
        """Latent tokens of the patches listed in `keep` (N x visible indices)."""
        patches = patchify(volume)
        visible = patches.gather(1, keep[..., None].expand(-1, -1, VOXELS))
        position = self.position.expand(len(patches), -1, -1)
        position = position.gather(1, keep[..., None].expand(-1, -1, position.shape[-1]))
        return self.encoder_norm(self.encoder(self.embedding(visible) + position))

    def forward(self, volume, noise=None):
        """Mask random patches; return the reconstructed volume and the mask volume (1 = hidden)."""
        batch = volume.shape[0]
        if noise is None:
            noise = torch.rand(batch, TOKENS, device=volume.device)
        order = noise.argsort(dim=1)
        restore = order.argsort(dim=1)
        encoded = self.to_decoder(self.encode(volume, order[:, : self.visible]))
        missing = self.mask_token.expand(batch, TOKENS - self.visible, -1)
        tokens = torch.cat([encoded, missing], dim=1)
        tokens = tokens.gather(1, restore[..., None].expand(-1, -1, tokens.shape[-1]))
        predicted = self.prediction(self.decoder_norm(self.decoder(tokens + self.decoder_position)))
        mask = torch.ones(batch, TOKENS, device=volume.device)
        mask[:, : self.visible] = 0
        mask = mask.gather(1, restore)
        return unpatchify(predicted), unpatchify(mask[..., None].expand(-1, -1, VOXELS))