davda54 commited on
Commit
fc8131f
·
verified ·
1 Parent(s): f04e0e8
Files changed (1) hide show
  1. modeling_gptbert.py +10 -6
modeling_gptbert.py CHANGED
@@ -593,15 +593,19 @@ class GptBertPreTrainedModel(PreTrainedModel):
593
  def _init_weights(self, module):
594
  std = math.sqrt(2.0 / (5.0 * self.hidden_size))
595
 
596
- if isinstance(module, nn.Linear) or isinstance(module, CastedLinearIn):
597
- nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
598
- if module.bias is not None:
599
- module.bias.data.zero_()
600
- elif isinstance(module, nn.Embedding):
601
  nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
602
  elif isinstance(module, nn.LayerNorm):
 
 
 
 
603
  module.bias.data.zero_()
604
- module.weight.data.fill_(1.0)
 
605
 
606
 
607
  class GptBertModel(GptBertPreTrainedModel):
 
593
  def _init_weights(self, module):
594
  std = math.sqrt(2.0 / (5.0 * self.hidden_size))
595
 
596
+ if isinstance(module, MultiCastedLinearOrthoIn):
597
+ for weight in module.weights:
598
+ nn.init.trunc_normal_(weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
599
+ elif isinstance(module, (nn.Linear, nn.Embedding)):
 
600
  nn.init.trunc_normal_(module.weight.data, mean=0.0, std=std, a=-2*std, b=2*std)
601
  elif isinstance(module, nn.LayerNorm):
602
+ if module.weight is not None:
603
+ module.weight.data.fill_(1.0)
604
+
605
+ if hasattr(module, 'bias') and module.bias is not None:
606
  module.bias.data.zero_()
607
+ if hasattr(module, 'scale') and isinstance(module.scale, nn.Parameter):
608
+ module.scale.data.fill_(1.0)
609
 
610
 
611
  class GptBertModel(GptBertPreTrainedModel):