summer summer commited on
Commit
7b4b2ed
·
verified ·
1 Parent(s): 0aabfc1

Upload modeling_loop_gqa_action.py

Browse files
Files changed (1) hide show
  1. modeling_loop_gqa_action.py +4 -1
modeling_loop_gqa_action.py CHANGED
@@ -144,7 +144,10 @@ class SwiGLU(nn.Module):
144
  def __init__(self, config):
145
  super().__init__()
146
 
147
- hidden_dim = int(config.d_model * config.mlp_ratio)
 
 
 
148
 
149
  self.gate_proj = nn.Linear(
150
  config.d_model,
 
144
  def __init__(self, config):
145
  super().__init__()
146
 
147
+ # Checkpoint uses 1536 for d_model=512.
148
+ # Keep the exact trained intermediate size instead of truncating
149
+ # d_model * mlp_ratio (which would incorrectly produce 1365).
150
+ hidden_dim = getattr(config, "intermediate_size", 1536)
151
 
152
  self.gate_proj = nn.Linear(
153
  config.d_model,