summer summer commited on
Upload modeling_loop_gqa_action.py
Browse files
modeling_loop_gqa_action.py
CHANGED
|
@@ -144,7 +144,10 @@ class SwiGLU(nn.Module):
|
|
| 144 |
def __init__(self, config):
|
| 145 |
super().__init__()
|
| 146 |
|
| 147 |
-
|
|
|
|
|
|
|
|
|
|
| 148 |
|
| 149 |
self.gate_proj = nn.Linear(
|
| 150 |
config.d_model,
|
|
|
|
| 144 |
def __init__(self, config):
|
| 145 |
super().__init__()
|
| 146 |
|
| 147 |
+
# Checkpoint uses 1536 for d_model=512.
|
| 148 |
+
# Keep the exact trained intermediate size instead of truncating
|
| 149 |
+
# d_model * mlp_ratio (which would incorrectly produce 1365).
|
| 150 |
+
hidden_dim = getattr(config, "intermediate_size", 1536)
|
| 151 |
|
| 152 |
self.gate_proj = nn.Linear(
|
| 153 |
config.d_model,
|