summer summer commited on
Commit
a8ccdcd
·
verified ·
1 Parent(s): 98a7259

Upload 2 files

Browse files
Files changed (2) hide show
  1. modeling_loop_gqa_action.py +8 -8
  2. test_inference.py +11 -4
modeling_loop_gqa_action.py CHANGED
@@ -457,9 +457,14 @@ class LoopTransformerLM(
457
  self.token_embedding.weight
458
  )
459
 
460
- # Exact original initialization.
461
- self.apply(
462
- self._init_weights
 
 
 
 
 
463
  )
464
 
465
  def _init_weights(
@@ -503,11 +508,6 @@ class LoopTransformerLM(
503
  ):
504
  self.lm_head = new_embeddings
505
 
506
- def tie_weights(self):
507
- # Preserve the original exact shared parameter.
508
- self.lm_head.weight = (
509
- self.token_embedding.weight
510
- )
511
 
512
  def forward(
513
  self,
 
457
  self.token_embedding.weight
458
  )
459
 
460
+ # Initialize Transformers bookkeeping for modern versions.
461
+ # This sets `all_tied_weights_keys` and performs standard
462
+ # PreTrainedModel post-initialization needed by from_pretrained().
463
+ self.post_init()
464
+
465
+ # Re-assert the exact shared parameter after post_init.
466
+ self.lm_head.weight = (
467
+ self.token_embedding.weight
468
  )
469
 
470
  def _init_weights(
 
508
  ):
509
  self.lm_head = new_embeddings
510
 
 
 
 
 
 
511
 
512
  def forward(
513
  self,
test_inference.py CHANGED
@@ -13,10 +13,17 @@ model = AutoModelForCausalLM.from_pretrained(
13
  trust_remote_code=True,
14
  device_map="auto",
15
  )
 
16
  model.eval()
17
 
18
- print("model:", type(model).__name__)
19
- print("tied:", model.lm_head.weight.data_ptr() == model.token_embedding.weight.data_ptr())
 
 
 
 
 
 
20
 
21
  prompt = "Task: Open Chrome and search for OpenAI.\nAction:"
22
 
@@ -26,7 +33,7 @@ inputs = tokenizer(
26
  ).to(model.device)
27
 
28
  with torch.no_grad():
29
- out = model.generate(
30
  **inputs,
31
  max_new_tokens=64,
32
  do_sample=False,
@@ -34,4 +41,4 @@ with torch.no_grad():
34
  pad_token_id=tokenizer.eos_token_id,
35
  )
36
 
37
- print(tokenizer.decode(out[0], skip_special_tokens=True))
 
13
  trust_remote_code=True,
14
  device_map="auto",
15
  )
16
+
17
  model.eval()
18
 
19
+ print("class:", type(model).__name__)
20
+ print("has all_tied_weights_keys:", hasattr(model, "all_tied_weights_keys"))
21
+ print("all_tied_weights_keys:", getattr(model, "all_tied_weights_keys", None))
22
+ print(
23
+ "embedding tied:",
24
+ model.token_embedding.weight.data_ptr()
25
+ == model.lm_head.weight.data_ptr(),
26
+ )
27
 
28
  prompt = "Task: Open Chrome and search for OpenAI.\nAction:"
29
 
 
33
  ).to(model.device)
34
 
35
  with torch.no_grad():
36
+ output = model.generate(
37
  **inputs,
38
  max_new_tokens=64,
39
  do_sample=False,
 
41
  pad_token_id=tokenizer.eos_token_id,
42
  )
43
 
44
+ print(tokenizer.decode(output[0], skip_special_tokens=True))