import torch from torch import nn from ..modules import (ConvSC, ConvNeXtSubBlock, ConvMixerSubBlock, GASubBlock, gInception_ST, HorNetSubBlock, MLPMixerSubBlock, MogaSubBlock, PoolFormerSubBlock, SwinSubBlock, UniformerSubBlock, VANSubBlock, ViTSubBlock, TAUSubBlock) class SimVP_Model(nn.Module): r"""SimVP Model Implementation of `SimVP: Simpler yet Better Video Prediction `_. """ def __init__(self, in_shape, hid_S=16, hid_T=256, N_S=4, N_T=4, model_type='gSTA', mlp_ratio=8., drop=0.0, drop_path=0.0, spatio_kernel_enc=3, spatio_kernel_dec=3, act_inplace=True, **kwargs): super(SimVP_Model, self).__init__() T, C, H, W = in_shape # T is pre_seq_length self.out_shape = kwargs.get('out_shape', (T,C,H,W)) print('out_shape', self.out_shape) kwargs['enc_C']=hid_S H, W = int(H / 2**(N_S/2)), int(W / 2**(N_S/2)) # downsample 1 / 2**(N_S/2) act_inplace = False self.enc = Encoder(C, hid_S, N_S, spatio_kernel_enc, act_inplace=act_inplace) self.dec = Decoder(hid_S, self.out_shape[1], N_S, spatio_kernel_dec, act_inplace=act_inplace) model_type = 'gsta' if model_type is None else model_type.lower() if model_type == 'incepu': self.hid = MidIncepNet(T*hid_S, hid_T, N_T) else: self.hid = MidMetaNet(T*hid_S, hid_T, N_T, input_resolution=(H, W), model_type=model_type, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, **kwargs) if T != self.out_shape[0]: self.skipconv = nn.Sequential( nn.Conv2d(T*hid_S,self.out_shape[0]*hid_S,7, padding='same'), nn.SiLU() ) #self.dropout = nn.Dropout(0.4) #self.isdrop = kwargs.get('isdrop', False) #print(self.isdrop) def forward(self, x_raw, **kwargs): B, T, C, H, W = x_raw.shape x = x_raw.view(B*T, C, H, W) embed, skip = self.enc(x) _, C_, H_, W_ = embed.shape z = embed.view(B, T, C_, H_, W_) hid = self.hid(z) hid = hid.reshape(B*self.out_shape[0], C_, H_, W_) #if self.isdrop: hid = self.dropout(hid) if hid.shape[0] != skip.shape[0]: skip = skip.view(B,T,C_,H,W) skip = skip.view(B,T*C_, H,W) skip = self.skipconv(skip) skip = skip.view(B*self.out_shape[0],C_,H,W) Y = self.dec(hid, skip) Y = Y.reshape(B, self.out_shape[0], self.out_shape[1], H, W) Y = Y.reshape(B,H,W) return torch.sigmoid(Y) def sampling_generator(N, reverse=False): samplings = [False, True] * (N // 2) if reverse: return list(reversed(samplings[:N])) else: return samplings[:N] class Encoder(nn.Module): """3D Encoder for SimVP""" def __init__(self, C_in, C_hid, N_S, spatio_kernel, act_inplace=True): samplings = sampling_generator(N_S) super(Encoder, self).__init__() self.enc = nn.Sequential( ConvSC(C_in, C_hid, spatio_kernel, downsampling=samplings[0], act_inplace=act_inplace), *[ConvSC(C_hid, C_hid, spatio_kernel, downsampling=s, act_inplace=act_inplace) for s in samplings[1:]] ) def forward(self, x): # B*4, 3, 128, 128 enc1 = self.enc[0](x) latent = enc1 for i in range(1, len(self.enc)): latent = self.enc[i](latent) return latent, enc1 import torch.nn.functional as F class Decoder(nn.Module): """3D Decoder for SimVP""" def __init__(self, C_hid, C_out, N_S, spatio_kernel, act_inplace=True, **kwargs): samplings = sampling_generator(N_S, reverse=True) super(Decoder, self).__init__() self.dec = nn.Sequential( *[ConvSC(C_hid, C_hid, spatio_kernel, upsampling=s, act_inplace=act_inplace) for s in samplings[:-1]], ConvSC(C_hid*2, C_hid, spatio_kernel, upsampling=samplings[-1], act_inplace=act_inplace) ) first_c = max(C_hid//3*2,1) second_c = max(C_hid//3, 1) self.readout = nn.Sequential( nn.Conv2d(C_hid, first_c, 3, padding='same'), nn.SiLU(), nn.Conv2d(first_c, second_c, 3,padding='same'), nn.SiLU(), nn.Conv2d(second_c, C_out, 3,padding='same') ) #self.readout = nn.Conv2d(C_hid, C_out, 1) def forward(self, hid, enc1=None): for i in range(0, len(self.dec)-1): hid = self.dec[i](hid) if hid.shape[-2:] != enc1.shape[-2:]: #print(hid.shape) hid = F.interpolate(hid, size=(enc1.shape[2], enc1.shape[3]), mode='bilinear', align_corners=False) #Y = self.dec[-1](hid + enc1) Y = self.dec[-1](combined) Y = self.readout(Y) return Y class MidIncepNet(nn.Module): """The hidden Translator of IncepNet for SimVPv1""" def __init__(self, channel_in, channel_hid, N2, incep_ker=[3,5,7,11], groups=8, **kwargs): super(MidIncepNet, self).__init__() assert N2 >= 2 and len(incep_ker) > 1 self.N2 = N2 enc_layers = [gInception_ST( channel_in, channel_hid//2, channel_hid, incep_ker= incep_ker, groups=groups)] for i in range(1,N2-1): enc_layers.append( gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) enc_layers.append( gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) dec_layers = [ gInception_ST(channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)] for i in range(1,N2-1): dec_layers.append( gInception_ST(2*channel_hid, channel_hid//2, channel_hid, incep_ker=incep_ker, groups=groups)) dec_layers.append( gInception_ST(2*channel_hid, channel_hid//2, channel_in, incep_ker=incep_ker, groups=groups)) self.enc = nn.Sequential(*enc_layers) self.dec = nn.Sequential(*dec_layers) def forward(self, x): B, T, C, H, W = x.shape x = x.reshape(B, T*C, H, W) # encoder skips = [] z = x for i in range(self.N2): z = self.enc[i](z) if i < self.N2-1: skips.append(z) # decoder z = self.dec[0](z) for i in range(1,self.N2): z = self.dec[i](torch.cat([z, skips[-i]], dim=1) ) y = z.reshape(B, T, C, H, W) return y class MetaBlock(nn.Module): """The hidden Translator of MetaFormer for SimVP""" def __init__(self, in_channels, out_channels, input_resolution=None, model_type=None, mlp_ratio=8., drop=0.0, drop_path=0.0, layer_i=0): super(MetaBlock, self).__init__() self.in_channels = in_channels self.out_channels = out_channels model_type = model_type.lower() if model_type is not None else 'gsta' if model_type == 'gsta': self.block = GASubBlock( in_channels, kernel_size=21, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, act_layer=nn.GELU) elif model_type == 'convmixer': self.block = ConvMixerSubBlock(in_channels, kernel_size=11, activation=nn.GELU) elif model_type == 'convnext': self.block = ConvNeXtSubBlock( in_channels, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) elif model_type == 'hornet': self.block = HorNetSubBlock(in_channels, mlp_ratio=mlp_ratio, drop_path=drop_path) elif model_type in ['mlp', 'mlpmixer']: self.block = MLPMixerSubBlock( in_channels, input_resolution, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) elif model_type in ['moga', 'moganet']: self.block = MogaSubBlock( in_channels, mlp_ratio=mlp_ratio, drop_rate=drop, drop_path_rate=drop_path) elif model_type == 'poolformer': self.block = PoolFormerSubBlock( in_channels, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) elif model_type == 'swin': self.block = SwinSubBlock( in_channels, input_resolution, layer_i=layer_i, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) elif model_type == 'uniformer': block_type = 'MHSA' if in_channels == out_channels and layer_i > 0 else 'Conv' self.block = UniformerSubBlock( in_channels, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, block_type=block_type) elif model_type == 'van': self.block = VANSubBlock( in_channels, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, act_layer=nn.GELU) elif model_type == 'vit': self.block = ViTSubBlock( in_channels, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path) elif model_type == 'tau': self.block = TAUSubBlock( in_channels, kernel_size=21, mlp_ratio=mlp_ratio, drop=drop, drop_path=drop_path, act_layer=nn.GELU) else: assert False and "Invalid model_type in SimVP" if in_channels != out_channels: self.reduction = nn.Conv2d( in_channels, out_channels, kernel_size=1, stride=1, padding=0) def forward(self, x): z = self.block(x) return z if self.in_channels == self.out_channels else self.reduction(z) class MidMetaNet(nn.Module): """The hidden Translator of MetaFormer for SimVP""" def __init__(self, channel_in, channel_hid, N2, input_resolution=None, model_type=None, mlp_ratio=4., drop=0.0, drop_path=0.1, **kwargs): super(MidMetaNet, self).__init__() assert N2 >= 2 and mlp_ratio > 1 self.N2 = N2 self.out_shape = kwargs.get('out_shape', None) self.enc_C = kwargs.get('enc_C', None) dpr = [ # stochastic depth decay rule x.item() for x in torch.linspace(1e-2, drop_path, self.N2)] # downsample enc_layers = [MetaBlock( channel_in, channel_hid, input_resolution, model_type, mlp_ratio, drop, drop_path=dpr[0], layer_i=0)] # middle layers for i in range(1, N2-1): enc_layers.append(MetaBlock( channel_hid, channel_hid, input_resolution, model_type, mlp_ratio, drop, drop_path=dpr[i], layer_i=i)) # upsample enc_layers.append(MetaBlock( channel_hid, channel_in if self.out_shape is None else self.out_shape[0]*self.enc_C, input_resolution, model_type, mlp_ratio, drop, drop_path=drop_path, layer_i=N2-1)) self.enc = nn.Sequential(*enc_layers) def forward(self, x): B, T, C, H, W = x.shape x = x.reshape(B, T*C, H, W) z = x for i in range(self.N2): z = self.enc[i](z) y = z.reshape(B, T if self.out_shape is None else self.out_shape[0], C, H, W) return y