Update normalizer.py
Browse files- normalizer.py +4 -4
normalizer.py
CHANGED
|
@@ -26,8 +26,8 @@ class NormalizerBlock(nn.Module):
|
|
| 26 |
def __init__(self, d_model, d_ffn, num_tokens):
|
| 27 |
super().__init__()
|
| 28 |
|
| 29 |
-
self.norm_global = nn.LayerNorm(d_model * num_tokens, elementwise_affine=False)
|
| 30 |
-
self.norm_local = nn.LayerNorm(d_model, elementwise_affine=False)
|
| 31 |
self.gating = GatingUnit(d_model, d_ffn)
|
| 32 |
|
| 33 |
def forward(self, x):
|
|
@@ -37,10 +37,10 @@ class NormalizerBlock(nn.Module):
|
|
| 37 |
dim0 = x.shape[0]
|
| 38 |
dim1 = x.shape[1]
|
| 39 |
dim2 = x.shape[2]
|
| 40 |
-
x = x.reshape([dim0,dim1*dim2])
|
| 41 |
x = self.norm_global(x)
|
| 42 |
|
| 43 |
-
x = x.reshape([dim0,dim1,dim2])
|
| 44 |
x = x + residual
|
| 45 |
|
| 46 |
residual = x
|
|
|
|
| 26 |
def __init__(self, d_model, d_ffn, num_tokens):
|
| 27 |
super().__init__()
|
| 28 |
|
| 29 |
+
self.norm_global = nn.LayerNorm(d_model * num_tokens, elementwise_affine = False)
|
| 30 |
+
self.norm_local = nn.LayerNorm(d_model, elementwise_affine = False)
|
| 31 |
self.gating = GatingUnit(d_model, d_ffn)
|
| 32 |
|
| 33 |
def forward(self, x):
|
|
|
|
| 37 |
dim0 = x.shape[0]
|
| 38 |
dim1 = x.shape[1]
|
| 39 |
dim2 = x.shape[2]
|
| 40 |
+
x = x.reshape([dim0, dim1 * dim2])
|
| 41 |
x = self.norm_global(x)
|
| 42 |
|
| 43 |
+
x = x.reshape([dim0, dim1, dim2])
|
| 44 |
x = x + residual
|
| 45 |
|
| 46 |
residual = x
|