summer summer commited on
Commit
971c0d6
·
verified ·
1 Parent(s): 9bae26c

Update modeling_loop_gqa_action.py

Browse files
Files changed (1) hide show
  1. modeling_loop_gqa_action.py +16 -14
modeling_loop_gqa_action.py CHANGED
@@ -409,6 +409,12 @@ class LoopTransformerLM(
409
  def __init__(self, config):
410
  super().__init__(config)
411
 
 
 
 
 
 
 
412
  assert (
413
  config.n_heads
414
  % config.n_kv_heads
@@ -457,11 +463,20 @@ class LoopTransformerLM(
457
  self.token_embedding.weight
458
  )
459
 
460
- # Exact original initialization.
 
461
  self.apply(
462
  self._init_weights
463
  )
464
 
 
 
 
 
 
 
 
 
465
  def _init_weights(
466
  self,
467
  module,
@@ -611,16 +626,3 @@ class LoopTransformerLM(
611
  "input_ids": input_ids,
612
  "attention_mask": attention_mask,
613
  }
614
- # Compatibility alias for Hub configs that reference
615
- # modeling_loop_gqa_action.LoopGQAActionModel.
616
- class LoopGQAActionModel(LoopTransformerLM):
617
- config_class = LoopGQAActionConfig
618
- _tied_weights_keys = {
619
- "lm_head.weight": "token_embedding.weight"
620
- }
621
-
622
- def __init__(self, config):
623
- # LoopTransformerLM.__init__ already calls PreTrainedModel.__init__
624
- # and post_init(), so modern Transformers bookkeeping including
625
- # all_tied_weights_keys is initialized correctly.
626
- super().__init__(config)
 
409
  def __init__(self, config):
410
  super().__init__(config)
411
 
412
+ # Transformers compatibility (newer versions inspect this during
413
+ # from_pretrained finalization). Define it eagerly.
414
+ self.all_tied_weights_keys = {
415
+ "lm_head.weight": "token_embedding.weight"
416
+ }
417
+
418
  assert (
419
  config.n_heads
420
  % config.n_kv_heads
 
463
  self.token_embedding.weight
464
  )
465
 
466
+ # Initialize using the original rule, then initialize
467
+ # Transformers bookkeeping required by modern releases.
468
  self.apply(
469
  self._init_weights
470
  )
471
 
472
+ self.post_init()
473
+
474
+ # Reassert exact tied parameter after post_init.
475
+ self.lm_head.weight = self.token_embedding.weight
476
+ self.all_tied_weights_keys = {
477
+ "lm_head.weight": "token_embedding.weight"
478
+ }
479
+
480
  def _init_weights(
481
  self,
482
  module,
 
626
  "input_ids": input_ids,
627
  "attention_mask": attention_mask,
628
  }