Download model.py from EScO-V10/MedSegNet: direct link, hf CLI and curl.
- Browser
- Download file 19.8 kB
-
https://huggingface.co/spaces/EScO-V10/MedSegNet/resolve/main/model.py
- Command line
-
hf download hf://spaces/EScO-V10/MedSegNet/model.py
-
curl -L -o model.py https://huggingface.co/spaces/EScO-V10/MedSegNet/resolve/main/model.py
19.8 kB
| import torch | |
| import math | |
| from torch import nn | |
| def compute_cube_root(n: int) -> int: | |
| """Compute integer cube root efficiently.""" | |
| return int(round(n ** (1.0 / 3.0))) | |
| class PatchEmbedding3D(nn.Module): | |
| def __init__( | |
| self, | |
| in_channel: int = 4, | |
| embed_dim: int = 768, | |
| kernel_size: int = 7, | |
| stride: int = 4, | |
| padding: int = 3, | |
| ): | |
| """ | |
| in_channels: number of the channels in the input volume | |
| embed_dim: embedding dimmesion of the patch | |
| """ | |
| super().__init__() | |
| self.patch_embeddings = nn.Conv3d( | |
| in_channel, | |
| embed_dim, | |
| kernel_size=kernel_size, | |
| stride=stride, | |
| padding=padding, | |
| ) | |
| self.norm = nn.LayerNorm(embed_dim) | |
| def forward(self, x): | |
| # standard embedding patch | |
| patches = self.patch_embeddings(x) | |
| patches = patches.flatten(2).transpose(1, 2) | |
| patches = self.norm(patches) | |
| return patches | |
| class SelfAttention(nn.Module): | |
| def __init__( | |
| self, | |
| embed_dim: int = 768, | |
| num_heads: int = 8, | |
| sr_ratio: int = 2, | |
| qkv_bias: bool = False, | |
| attn_dropout: float = 0.0, | |
| proj_dropout: float = 0.0, | |
| ): | |
| super().__init__() | |
| assert ( | |
| embed_dim % num_heads == 0 | |
| ), "Embedding dim should be divisible by number of heads!" | |
| self.num_heads = num_heads | |
| self.embed_dim = embed_dim | |
| # embedding dimesion of each attention head | |
| self.attention_head_dim = embed_dim // num_heads | |
| self.scale = self.attention_head_dim ** -0.5 | |
| # The same input is used to generate the query, key, and value, | |
| # (batch_size, num_patches, hidden_size) -> (batch_size, num_patches, attention_head_size) | |
| self.query = nn.Linear(embed_dim, embed_dim, bias=qkv_bias) | |
| self.key_value = nn.Linear(embed_dim, 2 * embed_dim, bias=qkv_bias) | |
| self.attn_dropout_p = attn_dropout | |
| self.proj = nn.Linear(embed_dim, embed_dim) | |
| self.proj_dropout = nn.Dropout(proj_dropout) | |
| self.sr_ratio = sr_ratio | |
| if sr_ratio > 1: | |
| self.sr = nn.Conv3d( | |
| embed_dim, embed_dim, kernel_size=sr_ratio, stride=sr_ratio | |
| ) | |
| self.sr_norm = nn.LayerNorm(embed_dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| # (batch_size, num_patches, hidden_size) | |
| B, N, C = x.shape | |
| # (batch_size, num_head, sequence_length, embed_dim) | |
| q = ( | |
| self.query(x) | |
| .reshape(B, N, self.num_heads, self.attention_head_dim) | |
| .permute(0, 2, 1, 3) | |
| .contiguous() | |
| ) | |
| if self.sr_ratio > 1: | |
| n = compute_cube_root(N) | |
| # (batch_size, sequence_length, embed_dim) -> (batch_size, embed_dim, patch_D, patch_H, patch_W) | |
| x_ = x.permute(0, 2, 1).reshape(B, C, n, n, n).contiguous() | |
| # (batch_size, embed_dim, patch_D, patch_H, patch_W) -> (batch_size, embed_dim, patch_D/sr_ratio, patch_H/sr_ratio, patch_W/sr_ratio) | |
| x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1).contiguous() | |
| # (batch_size, embed_dim, patch_D/sr_ratio, patch_H/sr_ratio, patch_W/sr_ratio) -> (batch_size, sequence_length, embed_dim) | |
| # normalizing the layer | |
| x_ = self.sr_norm(x_) | |
| # (batch_size, num_patches, hidden_size) | |
| kv = ( | |
| self.key_value(x_) | |
| .reshape(B, -1, 2, self.num_heads, self.attention_head_dim) | |
| .permute(2, 0, 3, 1, 4) | |
| ) | |
| # (2, batch_size, num_heads, num_sequence, attention_head_dim) | |
| else: | |
| # (batch_size, num_patches, hidden_size) | |
| kv = ( | |
| self.key_value(x) | |
| .reshape(B, -1, 2, self.num_heads, self.attention_head_dim) | |
| .permute(2, 0, 3, 1, 4) | |
| ) | |
| # (2, batch_size, num_heads, num_sequence, attention_head_dim) | |
| k, v = kv[0].contiguous(), kv[1].contiguous() | |
| # Use PyTorch's optimized scaled_dot_product_attention when available (PyTorch 2.0+) | |
| # Falls back to manual computation for older versions | |
| if hasattr(torch.nn.functional, 'scaled_dot_product_attention'): | |
| out = torch.nn.functional.scaled_dot_product_attention( | |
| q, k, v, | |
| attn_mask=None, | |
| dropout_p=self.attn_dropout_p if self.training else 0.0, | |
| is_causal=False | |
| ) | |
| else: | |
| attention_score = (q @ k.transpose(-2, -1)) * self.scale | |
| attnention_prob = attention_score.softmax(dim=-1) | |
| if self.training and self.attn_dropout_p > 0: | |
| attnention_prob = torch.nn.functional.dropout(attnention_prob, p=self.attn_dropout_p, training=True) | |
| out = attnention_prob @ v | |
| out = out.transpose(1, 2).reshape(B, N, C).contiguous() | |
| out = self.proj(out) | |
| out = self.proj_dropout(out) | |
| return out | |
| class DepthwiseConv3D(nn.Module): | |
| def __init__(self, dim=768): | |
| super().__init__() | |
| self.dwconv = nn.Conv3d(dim, dim, 3, 1, 1, bias=True, groups=dim) | |
| # added batchnorm (remove it ?) | |
| self.bn = nn.BatchNorm3d(dim) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| B, N, C = x.shape | |
| # (batch, patch_cube, hidden_size) -> (batch, hidden_size, D, H, W) | |
| # assuming D = H = W, i.e. cube root of the patch is an integer number! | |
| n = compute_cube_root(N) | |
| x = x.transpose(1, 2).reshape(B, C, n, n, n).contiguous() | |
| x = self.dwconv(x) | |
| # added batchnorm (remove it ?) | |
| x = self.bn(x) | |
| x = x.flatten(2).transpose(1, 2).contiguous() | |
| return x | |
| class MixFFN3D(nn.Module): | |
| def __init__(self, in_feature: int, mlp_ratio: int = 2, dropout: float = 0.0): | |
| super().__init__() | |
| out_feature = mlp_ratio * in_feature | |
| self.fc1 = nn.Linear(in_feature, out_feature) | |
| self.dwconv = DepthwiseConv3D(dim=out_feature) | |
| self.fc2 = nn.Linear(out_feature, in_feature) | |
| self.act_fn = nn.GELU() | |
| self.dropout = nn.Dropout(dropout) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| x = self.fc1(x) | |
| x = self.dwconv(x) | |
| x = self.act_fn(x) | |
| x = self.dropout(x) | |
| x = self.fc2(x) | |
| x = self.dropout(x) | |
| return x | |
| class LinearProjection3D(nn.Module): | |
| def __init__(self, in_feature: int, mlp_ratio: int = 2, dropout: float = 0.0): | |
| super().__init__() | |
| out_feature = mlp_ratio * in_feature | |
| self.fc1 = nn.Linear(in_feature, out_feature) | |
| self.dwconv = DepthwiseConv3D(dim=out_feature) | |
| self.fc2 = nn.Linear(out_feature, in_feature) | |
| self.act_fn = nn.GELU() | |
| self.dropout = nn.Dropout(dropout) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| x = self.fc1(x) | |
| x = self.dwconv(x) | |
| x = self.act_fn(x) | |
| x = self.dropout(x) | |
| x = self.fc2(x) | |
| x = self.dropout(x) | |
| return x | |
| class TransformerBlock(nn.Module): | |
| def __init__( | |
| self, | |
| embed_dim: int = 768, | |
| mlp_ratio: int = 2, | |
| num_heads: int = 8, | |
| sr_ratio: int = 2, | |
| qkv_bias: bool = False, | |
| attn_dropout: float = 0.0, | |
| proj_dropout: float = 0.0, | |
| ): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(embed_dim) | |
| self.attention = SelfAttention( | |
| embed_dim=embed_dim, | |
| num_heads=num_heads, | |
| sr_ratio=sr_ratio, | |
| qkv_bias=qkv_bias, | |
| attn_dropout=attn_dropout, | |
| proj_dropout=proj_dropout, | |
| ) | |
| self.norm2 = nn.LayerNorm(embed_dim) | |
| self.mlp = MixFFN3D(in_feature=embed_dim, mlp_ratio=mlp_ratio, dropout=0.0) | |
| def forward(self, x): | |
| x = x + self.attention(self.norm1(x)) | |
| x = x + self.mlp(self.norm2(x)) | |
| return x | |
| class Model_Encoder(nn.Module): | |
| def __init__( | |
| self, | |
| in_channels: int = 4, | |
| sr_ratios: list = [8, 4, 2, 1], | |
| embed_dims: list = [64, 128, 320, 512], | |
| patch_kernel_size: list = [7, 3, 3, 3], | |
| patch_stride: list = [4, 2, 2, 2], | |
| patch_padding: list = [3, 1, 1, 1], | |
| mlp_ratios: list = [2, 2, 2, 2], | |
| num_heads: list = [1, 2, 5, 8], | |
| depths: list = [2, 2, 2, 2], | |
| ): | |
| super().__init__() | |
| # patch embedding at different Pyramid level | |
| self.embed_1 = PatchEmbedding3D( | |
| in_channel=in_channels, | |
| embed_dim=embed_dims[0], | |
| kernel_size=patch_kernel_size[0], | |
| stride=patch_stride[0], | |
| padding=patch_padding[0], | |
| ) | |
| self.embed_2 = PatchEmbedding3D( | |
| in_channel=embed_dims[0], | |
| embed_dim=embed_dims[1], | |
| kernel_size=patch_kernel_size[1], | |
| stride=patch_stride[1], | |
| padding=patch_padding[1], | |
| ) | |
| self.embed_3 = PatchEmbedding3D( | |
| in_channel=embed_dims[1], | |
| embed_dim=embed_dims[2], | |
| kernel_size=patch_kernel_size[2], | |
| stride=patch_stride[2], | |
| padding=patch_padding[2], | |
| ) | |
| self.embed_4 = PatchEmbedding3D( | |
| in_channel=embed_dims[2], | |
| embed_dim=embed_dims[3], | |
| kernel_size=patch_kernel_size[3], | |
| stride=patch_stride[3], | |
| padding=patch_padding[3], | |
| ) | |
| # block 1 | |
| self.tf_block1 = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| embed_dim=embed_dims[0], | |
| num_heads=num_heads[0], | |
| mlp_ratio=mlp_ratios[0], | |
| sr_ratio=sr_ratios[0], | |
| qkv_bias=True, | |
| ) | |
| for _ in range(depths[0]) | |
| ] | |
| ) | |
| self.norm1 = nn.LayerNorm(embed_dims[0]) | |
| # block 2 | |
| self.tf_block2 = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| embed_dim=embed_dims[1], | |
| num_heads=num_heads[1], | |
| mlp_ratio=mlp_ratios[1], | |
| sr_ratio=sr_ratios[1], | |
| qkv_bias=True, | |
| ) | |
| for _ in range(depths[1]) | |
| ] | |
| ) | |
| self.norm2 = nn.LayerNorm(embed_dims[1]) | |
| # block 3 | |
| self.tf_block3 = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| embed_dim=embed_dims[2], | |
| num_heads=num_heads[2], | |
| mlp_ratio=mlp_ratios[2], | |
| sr_ratio=sr_ratios[2], | |
| qkv_bias=True, | |
| ) | |
| for _ in range(depths[2]) | |
| ] | |
| ) | |
| self.norm3 = nn.LayerNorm(embed_dims[2]) | |
| # block 4 | |
| self.tf_block4 = nn.ModuleList( | |
| [ | |
| TransformerBlock( | |
| embed_dim=embed_dims[3], | |
| num_heads=num_heads[3], | |
| mlp_ratio=mlp_ratios[3], | |
| sr_ratio=sr_ratios[3], | |
| qkv_bias=True, | |
| ) | |
| for _ in range(depths[3]) | |
| ] | |
| ) | |
| self.norm4 = nn.LayerNorm(embed_dims[3]) | |
| def forward(self, x): | |
| out = [] | |
| # at each stage these are the following mappings: | |
| # (batch_size, num_patches, hidden_state) | |
| # (num_patches,) -> (D, H, W) | |
| # (batch_size, num_patches, hidden_state) -> (batch_size, hidden_state, D, H, W) | |
| # stage 1 | |
| x = self.embed_1(x) | |
| B, N, C = x.shape | |
| n = compute_cube_root(N) | |
| for i, blk in enumerate(self.tf_block1): | |
| x = blk(x) | |
| x = self.norm1(x) | |
| # (B, N, C) -> (B, D, H, W, C) -> (B, C, D, H, W) | |
| x = x.reshape(B, n, n, n, -1).permute(0, 4, 1, 2, 3).contiguous() | |
| out.append(x) | |
| # stage 2 | |
| x = self.embed_2(x) | |
| B, N, C = x.shape | |
| n = compute_cube_root(N) | |
| for i, blk in enumerate(self.tf_block2): | |
| x = blk(x) | |
| x = self.norm2(x) | |
| # (B, N, C) -> (B, D, H, W, C) -> (B, C, D, H, W) | |
| x = x.reshape(B, n, n, n, -1).permute(0, 4, 1, 2, 3).contiguous() | |
| out.append(x) | |
| # stage 3 | |
| x = self.embed_3(x) | |
| B, N, C = x.shape | |
| n = compute_cube_root(N) | |
| for i, blk in enumerate(self.tf_block3): | |
| x = blk(x) | |
| x = self.norm3(x) | |
| # (B, N, C) -> (B, D, H, W, C) -> (B, C, D, H, W) | |
| x = x.reshape(B, n, n, n, -1).permute(0, 4, 1, 2, 3).contiguous() | |
| out.append(x) | |
| # stage 4 | |
| x = self.embed_4(x) | |
| B, N, C = x.shape | |
| n = compute_cube_root(N) | |
| for i, blk in enumerate(self.tf_block4): | |
| x = blk(x) | |
| x = self.norm4(x) | |
| # (B, N, C) -> (B, D, H, W, C) -> (B, C, D, H, W) | |
| x = x.reshape(B, n, n, n, -1).permute(0, 4, 1, 2, 3).contiguous() | |
| out.append(x) | |
| return out | |
| class LinearProjection(nn.Module): | |
| """ | |
| Linear Embedding | |
| """ | |
| def __init__(self, input_dim=2048, embed_dim=768): | |
| super().__init__() | |
| self.proj = nn.Linear(input_dim, embed_dim) | |
| self.bn = nn.LayerNorm(embed_dim) | |
| def forward(self, x): | |
| x = x.flatten(2).transpose(1, 2).contiguous() | |
| x = self.proj(x) | |
| # added batchnorm (remove it ?) | |
| x = self.bn(x) | |
| return x | |
| class Model_DecoderHead(nn.Module): | |
| """ | |
| SegFormer: Simple and Efficient Design for Semantic Segmentation with Transformers | |
| """ | |
| def __init__( | |
| self, | |
| input_feature_dims: list = [512, 320, 128, 64], | |
| decoder_head_embedding_dim: int = 256, | |
| num_classes: int = 3, | |
| dropout: float = 0.0, | |
| ): | |
| super().__init__() | |
| self.linear_c4 = LinearProjection( | |
| input_dim=input_feature_dims[0], | |
| embed_dim=decoder_head_embedding_dim, | |
| ) | |
| self.linear_c3 = LinearProjection( | |
| input_dim=input_feature_dims[1], | |
| embed_dim=decoder_head_embedding_dim, | |
| ) | |
| self.linear_c2 = LinearProjection( | |
| input_dim=input_feature_dims[2], | |
| embed_dim=decoder_head_embedding_dim, | |
| ) | |
| self.linear_c1 = LinearProjection( | |
| input_dim=input_feature_dims[3], | |
| embed_dim=decoder_head_embedding_dim, | |
| ) | |
| # convolution module to combine feature maps generated by the mlps | |
| self.linear_fuse = nn.Sequential( | |
| nn.Conv3d( | |
| in_channels=4 * decoder_head_embedding_dim, | |
| out_channels=decoder_head_embedding_dim, | |
| kernel_size=1, | |
| stride=1, | |
| bias=False, | |
| ), | |
| nn.BatchNorm3d(decoder_head_embedding_dim), | |
| nn.ReLU(), | |
| ) | |
| self.dropout = nn.Dropout(dropout) | |
| # final linear projection layer | |
| self.linear_pred = nn.Conv3d( | |
| decoder_head_embedding_dim, num_classes, kernel_size=1 | |
| ) | |
| # segformer decoder generates the final decoded feature map size at 1/4 of the original input volume size | |
| self.upsample_volume = nn.Upsample( | |
| scale_factor=4.0, mode="trilinear", align_corners=False | |
| ) | |
| def forward(self, c1, c2, c3, c4): | |
| ############## _MLP decoder on C1-C4 ########### | |
| n, _, _, _, _ = c4.shape | |
| _c4 = ( | |
| self.linear_c4(c4) | |
| .permute(0, 2, 1) | |
| .reshape(n, -1, c4.shape[2], c4.shape[3], c4.shape[4]) | |
| .contiguous() | |
| ) | |
| _c4 = torch.nn.functional.interpolate( | |
| _c4, | |
| size=c1.size()[2:], | |
| mode="trilinear", | |
| align_corners=False, | |
| ) | |
| _c3 = ( | |
| self.linear_c3(c3) | |
| .permute(0, 2, 1) | |
| .reshape(n, -1, c3.shape[2], c3.shape[3], c3.shape[4]) | |
| .contiguous() | |
| ) | |
| _c3 = torch.nn.functional.interpolate( | |
| _c3, | |
| size=c1.size()[2:], | |
| mode="trilinear", | |
| align_corners=False, | |
| ) | |
| _c2 = ( | |
| self.linear_c2(c2) | |
| .permute(0, 2, 1) | |
| .reshape(n, -1, c2.shape[2], c2.shape[3], c2.shape[4]) | |
| .contiguous() | |
| ) | |
| _c2 = torch.nn.functional.interpolate( | |
| _c2, | |
| size=c1.size()[2:], | |
| mode="trilinear", | |
| align_corners=False, | |
| ) | |
| _c1 = ( | |
| self.linear_c1(c1) | |
| .permute(0, 2, 1) | |
| .reshape(n, -1, c1.shape[2], c1.shape[3], c1.shape[4]) | |
| .contiguous() | |
| ) | |
| _c = self.linear_fuse(torch.cat([_c4, _c3, _c2, _c1], dim=1)) | |
| x = self.dropout(_c) | |
| x = self.linear_pred(x) | |
| x = self.upsample_volume(x) | |
| return x | |
| class MedSegNet(nn.Module): | |
| def __init__( | |
| self, | |
| in_channels: int = 4, | |
| sr_ratios: list = [4, 2, 1, 1], | |
| embed_dims: list = [32, 64, 160, 256], | |
| patch_kernel_size: list = [7, 3, 3, 3], | |
| patch_stride: list = [4, 2, 2, 2], | |
| patch_padding: list = [3, 1, 1, 1], | |
| mlp_ratios: list = [4, 4, 4, 4], | |
| num_heads: list = [1, 2, 5, 8], | |
| depths: list = [2, 2, 2, 2], | |
| decoder_head_embedding_dim: int = 256, | |
| num_classes: int = 3, | |
| decoder_dropout: float = 0.0, | |
| ): | |
| super().__init__() | |
| self.MedSegNet_encoder = Model_Encoder( | |
| in_channels=in_channels, | |
| sr_ratios=sr_ratios, | |
| embed_dims=embed_dims, | |
| patch_kernel_size=patch_kernel_size, | |
| patch_stride=patch_stride, | |
| patch_padding=patch_padding, | |
| mlp_ratios=mlp_ratios, | |
| num_heads=num_heads, | |
| depths=depths, | |
| ) | |
| # decoder takes in the feature maps in the reversed order | |
| reversed_embed_dims = embed_dims[::-1] | |
| self.MedSegNet_decoder = Model_DecoderHead( | |
| input_feature_dims=reversed_embed_dims, | |
| decoder_head_embedding_dim=decoder_head_embedding_dim, | |
| num_classes=num_classes, | |
| dropout=decoder_dropout, | |
| ) | |
| self.apply(self._init_weights) | |
| def _init_weights(self, m): | |
| if isinstance(m, nn.Linear): | |
| nn.init.trunc_normal_(m.weight, std=0.02) | |
| if isinstance(m, nn.Linear) and m.bias is not None: | |
| nn.init.constant_(m.bias, 0) | |
| elif isinstance(m, nn.LayerNorm): | |
| nn.init.constant_(m.bias, 0) | |
| nn.init.constant_(m.weight, 1.0) | |
| elif isinstance(m, nn.BatchNorm2d): | |
| nn.init.constant_(m.bias, 0) | |
| nn.init.constant_(m.weight, 1.0) | |
| elif isinstance(m, nn.BatchNorm3d): | |
| nn.init.constant_(m.bias, 0) | |
| nn.init.constant_(m.weight, 1.0) | |
| elif isinstance(m, nn.Conv2d): | |
| fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels | |
| fan_out //= m.groups | |
| m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) | |
| if m.bias is not None: | |
| m.bias.data.zero_() | |
| elif isinstance(m, nn.Conv3d): | |
| fan_out = m.kernel_size[0] * m.kernel_size[1] * m.kernel_size[2] * m.out_channels | |
| fan_out //= m.groups | |
| m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) | |
| if m.bias is not None: | |
| m.bias.data.zero_() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| # embedding the input | |
| x = self.MedSegNet_encoder(x) | |
| # # unpacking the embedded features generated by the transformer | |
| c1, c2, c3, c4 = x[0], x[1], x[2], x[3] | |
| # decoding the embedded features | |
| x = self.MedSegNet_decoder(c1, c2, c3, c4) | |
| return x |