renpas22 commited on
Commit
8268436
·
1 Parent(s): bb7ed44

Fix tokenize() calls to use actual tokenizer

Browse files
Files changed (1) hide show
  1. src/reasoning/step_level_cot.py +10 -3
src/reasoning/step_level_cot.py CHANGED
@@ -240,6 +240,12 @@ class StepLevelCoTTrainer:
240
 
241
  def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
242
  """Initialize newly added token embeddings with semantic information."""
 
 
 
 
 
 
243
  with torch.no_grad():
244
  input_embeddings = self.model.get_input_embeddings()
245
 
@@ -249,12 +255,12 @@ class StepLevelCoTTrainer:
249
  lm_head = self.model.lm_head
250
 
251
  for i, description in enumerate(reversed(descriptions), start=1):
252
- # Tokenize the description
253
- tokenized = self.tokenizer.tokenize(description)
254
  if not tokenized:
255
  continue
256
 
257
- tokenized_ids = self.tokenizer.convert_tokens_to_ids(tokenized)
258
 
259
  # Average embeddings of description tokens
260
  new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
@@ -266,6 +272,7 @@ class StepLevelCoTTrainer:
266
  lm_head.weight[-i, :] = new_lm_head.clone().detach()
267
 
268
  logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
 
269
 
270
  def load_step_dataset(
271
  self,
 
240
 
241
  def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
242
  """Initialize newly added token embeddings with semantic information."""
243
+ # Get actual tokenizer (handle both Tokenizer and Processor objects)
244
+ if hasattr(self.tokenizer, 'tokenizer'):
245
+ actual_tokenizer = self.tokenizer.tokenizer
246
+ else:
247
+ actual_tokenizer = self.tokenizer
248
+
249
  with torch.no_grad():
250
  input_embeddings = self.model.get_input_embeddings()
251
 
 
255
  lm_head = self.model.lm_head
256
 
257
  for i, description in enumerate(reversed(descriptions), start=1):
258
+ # Tokenize the description using actual tokenizer
259
+ tokenized = actual_tokenizer.tokenize(description)
260
  if not tokenized:
261
  continue
262
 
263
+ tokenized_ids = actual_tokenizer.convert_tokens_to_ids(tokenized)
264
 
265
  # Average embeddings of description tokens
266
  new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
 
272
  lm_head.weight[-i, :] = new_lm_head.clone().detach()
273
 
274
  logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
275
+
276
 
277
  def load_step_dataset(
278
  self,