Model weights, code, sample and model card
Browse files- .gitattributes +1 -0
- README.md +86 -0
- example.py +23 -0
- model.pt +3 -0
- model.py +80 -0
- reconstruction.png +3 -0
- sample.npy +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
reconstruction.png filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: cc-by-nc-sa-4.0
|
| 3 |
+
datasets: [Ahus-AIM/EchoXFlow]
|
| 4 |
+
library_name: pytorch
|
| 5 |
+
tags: [echocardiography, ultrasound, masked-autoencoder, 3d]
|
| 6 |
+
---
|
| 7 |
+
# 3D echocardiography masked autoencoder
|
| 8 |
+
|
| 9 |
+
Transformer masked autoencoder for single 3D echocardiography volumes in native beamspace
|
| 10 |
+
(radius x azimuth x elevation = 256 x 64 x 64, values in [0, 1]). It reconstructs the volume from the
|
| 11 |
+
25% of its 16 x 8 x 8 voxel patches that stay visible; the figure shows the central elevation slice.
|
| 12 |
+
|
| 13 |
+

|
| 14 |
+
|
| 15 |
+
## Usage
|
| 16 |
+
|
| 17 |
+
Requires `torch` and `matplotlib`. Download the repository and run the code below from inside it
|
| 18 |
+
(it is also saved as `example.py`). It loads the model and the sample, hides part of the input, and
|
| 19 |
+
saves the figure above.
|
| 20 |
+
|
| 21 |
+
```python
|
| 22 |
+
import matplotlib.pyplot as plt
|
| 23 |
+
import numpy as np
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
from model import EchoMAE3D
|
| 27 |
+
|
| 28 |
+
model = EchoMAE3D()
|
| 29 |
+
model.load_state_dict(torch.load("model.pt", weights_only=True))
|
| 30 |
+
model.eval()
|
| 31 |
+
|
| 32 |
+
volume = torch.from_numpy(np.load("sample.npy")).float()[None, None] / 255 # 1 x 1 x 256 x 64 x 64
|
| 33 |
+
torch.manual_seed(0)
|
| 34 |
+
with torch.no_grad():
|
| 35 |
+
reconstruction, mask = model(volume) # hides 75% of the 16 x 8 x 8 voxel patches
|
| 36 |
+
|
| 37 |
+
panels = {"Input": volume, "Masked": volume * (1 - mask), "Reconstruction": reconstruction}
|
| 38 |
+
figure, axes = plt.subplots(1, 3, figsize=(6, 4.4))
|
| 39 |
+
for axis, (title, image) in zip(axes, panels.items()):
|
| 40 |
+
axis.imshow(image[0, 0, :, :, 32], cmap="gray", vmin=0, vmax=1, aspect=0.5) # central elevation slice
|
| 41 |
+
axis.set_title(title)
|
| 42 |
+
axis.axis("off")
|
| 43 |
+
figure.tight_layout()
|
| 44 |
+
figure.savefig("reconstruction.png", dpi=150)
|
| 45 |
+
```
|
| 46 |
+
|
| 47 |
+
## Model
|
| 48 |
+
|
| 49 |
+
- 17.2 million parameters; 1,024 patch tokens, 75% random masking.
|
| 50 |
+
- Encoder: 8 blocks, width 384, 6 heads. Decoder: 4 blocks, width 192, 6 heads.
|
| 51 |
+
- Trained with L1 reconstruction over all voxels (AdamW). The weights are the checkpoint with the lowest
|
| 52 |
+
validation L1 (0.063).
|
| 53 |
+
- `model.encode(volume, keep)` returns the latent tokens of the selected patches.
|
| 54 |
+
|
| 55 |
+
## Files
|
| 56 |
+
|
| 57 |
+
| File | Content |
|
| 58 |
+
|---|---|
|
| 59 |
+
| `model.pt` | Model weights (state dict, 69 MB) |
|
| 60 |
+
| `model.py` | Model definition |
|
| 61 |
+
| `sample.npy` | One beamspace volume, `uint8`, 256 x 64 x 64 |
|
| 62 |
+
| `example.py` | The code above |
|
| 63 |
+
|
| 64 |
+
## Training data and citation
|
| 65 |
+
|
| 66 |
+
Trained on [EchoXFlow](https://huggingface.co/datasets/Ahus-AIM/EchoXFlow), a clinical echocardiography
|
| 67 |
+
dataset in native beamspace. `sample.npy` is one volume from its validation fold.
|
| 68 |
+
|
| 69 |
+
- Paper: [EchoXFlow: A Beamspace Echocardiography Dataset for Cardiac Motion, Flow, and Function](https://arxiv.org/abs/2605.05447)
|
| 70 |
+
- Dataset: https://huggingface.co/datasets/Ahus-AIM/EchoXFlow
|
| 71 |
+
- Tooling: https://github.com/Ahus-AIM/EchoXFlow
|
| 72 |
+
|
| 73 |
+
```bibtex
|
| 74 |
+
@misc{stenhede2026echoxflow,
|
| 75 |
+
title = {EchoXFlow: A Beamspace Echocardiography Dataset for Cardiac Motion, Flow, and Function},
|
| 76 |
+
author = {Stenhede, Elias and Sulkowska, Joanna and Orstad, Eivind Bj{\o}rkan and Schirmer, Henrik and Ranjbar, Arian},
|
| 77 |
+
year = {2026},
|
| 78 |
+
eprint = {2605.05447},
|
| 79 |
+
archivePrefix = {arXiv},
|
| 80 |
+
primaryClass = {cs.CV}
|
| 81 |
+
}
|
| 82 |
+
```
|
| 83 |
+
|
| 84 |
+
## License
|
| 85 |
+
|
| 86 |
+
CC BY-NC-SA 4.0, the same license as EchoXFlow, from which the weights and the sample are derived.
|
example.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import matplotlib.pyplot as plt
|
| 2 |
+
import numpy as np
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from model import EchoMAE3D
|
| 6 |
+
|
| 7 |
+
model = EchoMAE3D()
|
| 8 |
+
model.load_state_dict(torch.load("model.pt", weights_only=True))
|
| 9 |
+
model.eval()
|
| 10 |
+
|
| 11 |
+
volume = torch.from_numpy(np.load("sample.npy")).float()[None, None] / 255 # 1 x 1 x 256 x 64 x 64
|
| 12 |
+
torch.manual_seed(0)
|
| 13 |
+
with torch.no_grad():
|
| 14 |
+
reconstruction, mask = model(volume) # hides 75% of the 16 x 8 x 8 voxel patches
|
| 15 |
+
|
| 16 |
+
panels = {"Input": volume, "Masked": volume * (1 - mask), "Reconstruction": reconstruction}
|
| 17 |
+
figure, axes = plt.subplots(1, 3, figsize=(6, 4.4))
|
| 18 |
+
for axis, (title, image) in zip(axes, panels.items()):
|
| 19 |
+
axis.imshow(image[0, 0, :, :, 32], cmap="gray", vmin=0, vmax=1, aspect=0.5) # central elevation slice
|
| 20 |
+
axis.set_title(title)
|
| 21 |
+
axis.axis("off")
|
| 22 |
+
figure.tight_layout()
|
| 23 |
+
figure.savefig("reconstruction.png", dpi=150)
|
model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e8380609d273450810c9da59f9343cccb3884448a4a70292e13f636acb1e382d
|
| 3 |
+
size 68963135
|
model.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""3D echocardiography masked autoencoder (transformer).
|
| 2 |
+
|
| 3 |
+
Input: N x 1 x 256 x 64 x 64 beamspace volumes (radius x azimuth x elevation) in [0, 1].
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
from torch import nn
|
| 8 |
+
|
| 9 |
+
SHAPE = (256, 64, 64) # radius, azimuth, elevation
|
| 10 |
+
PATCH = (16, 8, 8)
|
| 11 |
+
GRID = (16, 8, 8)
|
| 12 |
+
TOKENS = 1024
|
| 13 |
+
VOXELS = 1024 # voxels per patch
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def patchify(volume):
|
| 17 |
+
(r, a, e), (pr, pa, pe) = GRID, PATCH
|
| 18 |
+
volume = volume.reshape(-1, 1, r, pr, a, pa, e, pe)
|
| 19 |
+
return volume.permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(-1, TOKENS, VOXELS)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def unpatchify(patches):
|
| 23 |
+
(r, a, e), (pr, pa, pe) = GRID, PATCH
|
| 24 |
+
patches = patches.reshape(-1, r, a, e, pr, pa, pe, 1)
|
| 25 |
+
return patches.permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(-1, 1, *SHAPE)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class Block(nn.Module):
|
| 29 |
+
def __init__(self, width, heads):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.norm1 = nn.LayerNorm(width)
|
| 32 |
+
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
| 33 |
+
self.norm2 = nn.LayerNorm(width)
|
| 34 |
+
self.mlp = nn.Sequential(nn.Linear(width, width * 4), nn.GELU(), nn.Linear(width * 4, width))
|
| 35 |
+
|
| 36 |
+
def forward(self, x):
|
| 37 |
+
normalized = self.norm1(x)
|
| 38 |
+
x = x + self.attention(normalized, normalized, normalized, need_weights=False)[0]
|
| 39 |
+
return x + self.mlp(self.norm2(x))
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class EchoMAE3D(nn.Module):
|
| 43 |
+
def __init__(self, width=384, depth=8, decoder_width=192, decoder_depth=4, mask_ratio=0.75):
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.visible = int(TOKENS * (1 - mask_ratio))
|
| 46 |
+
self.embedding = nn.Linear(VOXELS, width)
|
| 47 |
+
self.position = nn.Parameter(torch.zeros(1, TOKENS, width))
|
| 48 |
+
self.encoder = nn.Sequential(*[Block(width, 6) for _ in range(depth)])
|
| 49 |
+
self.encoder_norm = nn.LayerNorm(width)
|
| 50 |
+
self.to_decoder = nn.Linear(width, decoder_width)
|
| 51 |
+
self.mask_token = nn.Parameter(torch.zeros(1, 1, decoder_width))
|
| 52 |
+
self.decoder_position = nn.Parameter(torch.zeros(1, TOKENS, decoder_width))
|
| 53 |
+
self.decoder = nn.Sequential(*[Block(decoder_width, 6) for _ in range(decoder_depth)])
|
| 54 |
+
self.decoder_norm = nn.LayerNorm(decoder_width)
|
| 55 |
+
self.prediction = nn.Linear(decoder_width, VOXELS)
|
| 56 |
+
|
| 57 |
+
def encode(self, volume, keep):
|
| 58 |
+
"""Latent tokens of the patches listed in `keep` (N x visible indices)."""
|
| 59 |
+
patches = patchify(volume)
|
| 60 |
+
visible = patches.gather(1, keep[..., None].expand(-1, -1, VOXELS))
|
| 61 |
+
position = self.position.expand(len(patches), -1, -1)
|
| 62 |
+
position = position.gather(1, keep[..., None].expand(-1, -1, position.shape[-1]))
|
| 63 |
+
return self.encoder_norm(self.encoder(self.embedding(visible) + position))
|
| 64 |
+
|
| 65 |
+
def forward(self, volume, noise=None):
|
| 66 |
+
"""Mask random patches; return the reconstructed volume and the mask volume (1 = hidden)."""
|
| 67 |
+
batch = volume.shape[0]
|
| 68 |
+
if noise is None:
|
| 69 |
+
noise = torch.rand(batch, TOKENS, device=volume.device)
|
| 70 |
+
order = noise.argsort(dim=1)
|
| 71 |
+
restore = order.argsort(dim=1)
|
| 72 |
+
encoded = self.to_decoder(self.encode(volume, order[:, : self.visible]))
|
| 73 |
+
missing = self.mask_token.expand(batch, TOKENS - self.visible, -1)
|
| 74 |
+
tokens = torch.cat([encoded, missing], dim=1)
|
| 75 |
+
tokens = tokens.gather(1, restore[..., None].expand(-1, -1, tokens.shape[-1]))
|
| 76 |
+
predicted = self.prediction(self.decoder_norm(self.decoder(tokens + self.decoder_position)))
|
| 77 |
+
mask = torch.ones(batch, TOKENS, device=volume.device)
|
| 78 |
+
mask[:, : self.visible] = 0
|
| 79 |
+
mask = mask.gather(1, restore)
|
| 80 |
+
return unpatchify(predicted), unpatchify(mask[..., None].expand(-1, -1, VOXELS))
|
reconstruction.png
ADDED
|
Git LFS Details
|
sample.npy
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6033daf6a8f17a616ed0a834690e9ff4c00f3c6a7f2d32bb4bc651426de7d547
|
| 3 |
+
size 1048704
|