Abdullah-Nazhat commited on
Commit
473f867
·
verified ·
1 Parent(s): 60915ee

Update normalizer.py

Browse files
Files changed (1) hide show
  1. normalizer.py +4 -4
normalizer.py CHANGED
@@ -26,8 +26,8 @@ class NormalizerBlock(nn.Module):
26
  def __init__(self, d_model, d_ffn, num_tokens):
27
  super().__init__()
28
 
29
- self.norm_global = nn.LayerNorm(d_model * num_tokens, elementwise_affine=False)
30
- self.norm_local = nn.LayerNorm(d_model, elementwise_affine=False)
31
  self.gating = GatingUnit(d_model, d_ffn)
32
 
33
  def forward(self, x):
@@ -37,10 +37,10 @@ class NormalizerBlock(nn.Module):
37
  dim0 = x.shape[0]
38
  dim1 = x.shape[1]
39
  dim2 = x.shape[2]
40
- x = x.reshape([dim0,dim1*dim2])
41
  x = self.norm_global(x)
42
 
43
- x = x.reshape([dim0,dim1,dim2])
44
  x = x + residual
45
 
46
  residual = x
 
26
  def __init__(self, d_model, d_ffn, num_tokens):
27
  super().__init__()
28
 
29
+ self.norm_global = nn.LayerNorm(d_model * num_tokens, elementwise_affine = False)
30
+ self.norm_local = nn.LayerNorm(d_model, elementwise_affine = False)
31
  self.gating = GatingUnit(d_model, d_ffn)
32
 
33
  def forward(self, x):
 
37
  dim0 = x.shape[0]
38
  dim1 = x.shape[1]
39
  dim2 = x.shape[2]
40
+ x = x.reshape([dim0, dim1 * dim2])
41
  x = self.norm_global(x)
42
 
43
+ x = x.reshape([dim0, dim1, dim2])
44
  x = x + residual
45
 
46
  residual = x