add MLP layers
Browse files
build/torch-universal/MLP/__init__.py
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from . import layers
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
__all__ = [
|
| 5 |
+
"layers",
|
| 6 |
+
]
|
build/torch-universal/MLP/layers.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torch_npu
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
class MLP(torch.nn.Module):
|
| 6 |
+
def forward(self, hidden_state):
|
| 7 |
+
gate_up = torch.cat((self.gate_proj(hidden_state), self.up_proj(hidden_state)), dim=-1)
|
| 8 |
+
down_proj = self.down_proj(torch_npu.npu_swiglu(gate_up))
|
| 9 |
+
return down_proj
|