summer summer commited on
Update modeling_loop_gqa_action.py
Browse files- modeling_loop_gqa_action.py +16 -14
modeling_loop_gqa_action.py
CHANGED
|
@@ -409,6 +409,12 @@ class LoopTransformerLM(
|
|
| 409 |
def __init__(self, config):
|
| 410 |
super().__init__(config)
|
| 411 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 412 |
assert (
|
| 413 |
config.n_heads
|
| 414 |
% config.n_kv_heads
|
|
@@ -457,11 +463,20 @@ class LoopTransformerLM(
|
|
| 457 |
self.token_embedding.weight
|
| 458 |
)
|
| 459 |
|
| 460 |
-
#
|
|
|
|
| 461 |
self.apply(
|
| 462 |
self._init_weights
|
| 463 |
)
|
| 464 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 465 |
def _init_weights(
|
| 466 |
self,
|
| 467 |
module,
|
|
@@ -611,16 +626,3 @@ class LoopTransformerLM(
|
|
| 611 |
"input_ids": input_ids,
|
| 612 |
"attention_mask": attention_mask,
|
| 613 |
}
|
| 614 |
-
# Compatibility alias for Hub configs that reference
|
| 615 |
-
# modeling_loop_gqa_action.LoopGQAActionModel.
|
| 616 |
-
class LoopGQAActionModel(LoopTransformerLM):
|
| 617 |
-
config_class = LoopGQAActionConfig
|
| 618 |
-
_tied_weights_keys = {
|
| 619 |
-
"lm_head.weight": "token_embedding.weight"
|
| 620 |
-
}
|
| 621 |
-
|
| 622 |
-
def __init__(self, config):
|
| 623 |
-
# LoopTransformerLM.__init__ already calls PreTrainedModel.__init__
|
| 624 |
-
# and post_init(), so modern Transformers bookkeeping including
|
| 625 |
-
# all_tied_weights_keys is initialized correctly.
|
| 626 |
-
super().__init__(config)
|
|
|
|
| 409 |
def __init__(self, config):
|
| 410 |
super().__init__(config)
|
| 411 |
|
| 412 |
+
# Transformers compatibility (newer versions inspect this during
|
| 413 |
+
# from_pretrained finalization). Define it eagerly.
|
| 414 |
+
self.all_tied_weights_keys = {
|
| 415 |
+
"lm_head.weight": "token_embedding.weight"
|
| 416 |
+
}
|
| 417 |
+
|
| 418 |
assert (
|
| 419 |
config.n_heads
|
| 420 |
% config.n_kv_heads
|
|
|
|
| 463 |
self.token_embedding.weight
|
| 464 |
)
|
| 465 |
|
| 466 |
+
# Initialize using the original rule, then initialize
|
| 467 |
+
# Transformers bookkeeping required by modern releases.
|
| 468 |
self.apply(
|
| 469 |
self._init_weights
|
| 470 |
)
|
| 471 |
|
| 472 |
+
self.post_init()
|
| 473 |
+
|
| 474 |
+
# Reassert exact tied parameter after post_init.
|
| 475 |
+
self.lm_head.weight = self.token_embedding.weight
|
| 476 |
+
self.all_tied_weights_keys = {
|
| 477 |
+
"lm_head.weight": "token_embedding.weight"
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
def _init_weights(
|
| 481 |
self,
|
| 482 |
module,
|
|
|
|
| 626 |
"input_ids": input_ids,
|
| 627 |
"attention_mask": attention_mask,
|
| 628 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|