renpas22 commited on
Commit
b8bd3e8
·
1 Parent(s): e20135f

Fix tokenizer access for Processor objects

Browse files
Files changed (1) hide show
  1. src/reasoning/step_level_cot.py +9 -1
src/reasoning/step_level_cot.py CHANGED
@@ -208,8 +208,16 @@ class StepLevelCoTTrainer:
208
 
209
  logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
210
 
 
 
 
 
 
 
 
 
211
  # Add special tokens to tokenizer
212
- num_added_toks = self.tokenizer.add_special_tokens(
213
  {"additional_special_tokens": SPECIAL_TOKENS}
214
  )
215
 
 
208
 
209
  logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
210
 
211
+ # Get actual tokenizer (handle both Tokenizer and Processor objects)
212
+ if hasattr(self.tokenizer, 'tokenizer'):
213
+ # This is a processor, extract the tokenizer
214
+ actual_tokenizer = self.tokenizer.tokenizer
215
+ else:
216
+ # This is already a tokenizer
217
+ actual_tokenizer = self.tokenizer
218
+
219
  # Add special tokens to tokenizer
220
+ num_added_toks = actual_tokenizer.add_special_tokens(
221
  {"additional_special_tokens": SPECIAL_TOKENS}
222
  )
223