Zheliu commited on
Commit
0495f4e
·
1 Parent(s): bd6c226

add MLP layers

Browse files
build/torch-universal/MLP/__init__.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ from . import layers
2
+
3
+
4
+ __all__ = [
5
+ "layers",
6
+ ]
build/torch-universal/MLP/layers.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch_npu
3
+
4
+
5
+ class MLP(torch.nn.Module):
6
+ def forward(self, hidden_state):
7
+ gate_up = torch.cat((self.gate_proj(hidden_state), self.up_proj(hidden_state)), dim=-1)
8
+ down_proj = self.down_proj(torch_npu.npu_swiglu(gate_up))
9
+ return down_proj