aethertp commited on
Commit
b36a2d5
·
verified ·
1 Parent(s): 4db2d6b

Fix attention_mask signature in forward and prepare_inputs_for_generation

Browse files
Files changed (1) hide show
  1. modeling_picolm_v3.py +8 -5
modeling_picolm_v3.py CHANGED
@@ -91,7 +91,6 @@ class TransformerBlockV3(nn.Module):
91
  def forward(self, x, cos, sin, pass_idx=0):
92
  norm_attn = self.attn_norm_1 if pass_idx == 0 else self.attn_norm_2
93
  norm_mlp = self.mlp_norm_1 if pass_idx == 0 else self.mlp_norm_2
94
- # V2 HATASI DÜZELTİLDİ: Doğru Kalıntı Yolu
95
  x = x + self.attn(norm_attn(x), cos, sin)
96
  x = x + self.mlp(norm_mlp(x))
97
  return x
@@ -118,7 +117,8 @@ class PicoLMV3ForCausalLM(PreTrainedModel, GenerationMixin):
118
  def set_input_embeddings(self, value):
119
  self.tok_embeddings = value
120
 
121
- def forward(self, input_ids=None, labels=None, **kwargs):
 
122
  B, S = input_ids.shape
123
  x_low = self.tok_embeddings(input_ids)
124
  x = self.emb_proj(x_low)
@@ -129,7 +129,6 @@ class PicoLMV3ForCausalLM(PreTrainedModel, GenerationMixin):
129
  x = block(x, cos, sin, pass_idx=p_idx)
130
 
131
  h = self.final_norm(x)
132
- # Factorized Tied Embedding Çıkışı
133
  h_low = F.linear(h, self.emb_proj.weight.t())
134
  logits = F.linear(h_low, self.tok_embeddings.weight)
135
 
@@ -139,5 +138,9 @@ class PicoLMV3ForCausalLM(PreTrainedModel, GenerationMixin):
139
 
140
  return CausalLMOutputWithPast(loss=loss, logits=logits)
141
 
142
- def prepare_inputs_for_generation(self, input_ids, **kwargs):
143
- return {"input_ids": input_ids}
 
 
 
 
 
91
  def forward(self, x, cos, sin, pass_idx=0):
92
  norm_attn = self.attn_norm_1 if pass_idx == 0 else self.attn_norm_2
93
  norm_mlp = self.mlp_norm_1 if pass_idx == 0 else self.mlp_norm_2
 
94
  x = x + self.attn(norm_attn(x), cos, sin)
95
  x = x + self.mlp(norm_mlp(x))
96
  return x
 
117
  def set_input_embeddings(self, value):
118
  self.tok_embeddings = value
119
 
120
+ # attention_mask parametresi açıkça eklendi
121
+ def forward(self, input_ids=None, attention_mask=None, labels=None, past_key_values=None, **kwargs):
122
  B, S = input_ids.shape
123
  x_low = self.tok_embeddings(input_ids)
124
  x = self.emb_proj(x_low)
 
129
  x = block(x, cos, sin, pass_idx=p_idx)
130
 
131
  h = self.final_norm(x)
 
132
  h_low = F.linear(h, self.emb_proj.weight.t())
133
  logits = F.linear(h_low, self.tok_embeddings.weight)
134
 
 
138
 
139
  return CausalLMOutputWithPast(loss=loss, logits=logits)
140
 
141
+ # prepare_inputs_for_generation attention_mask'ı artık yakalıyor
142
+ def prepare_inputs_for_generation(self, input_ids, attention_mask=None, **kwargs):
143
+ model_inputs = {"input_ids": input_ids}
144
+ if attention_mask is not None:
145
+ model_inputs["attention_mask"] = attention_mask
146
+ return model_inputs