renpas22 commited on
Commit ·
b8bd3e8
1
Parent(s): e20135f
Fix tokenizer access for Processor objects
Browse files
src/reasoning/step_level_cot.py
CHANGED
|
@@ -208,8 +208,16 @@ class StepLevelCoTTrainer:
|
|
| 208 |
|
| 209 |
logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
# Add special tokens to tokenizer
|
| 212 |
-
num_added_toks =
|
| 213 |
{"additional_special_tokens": SPECIAL_TOKENS}
|
| 214 |
)
|
| 215 |
|
|
|
|
| 208 |
|
| 209 |
logger.info(f"Adding {len(SPECIAL_TOKENS)} special tokens for chain-of-thought reasoning...")
|
| 210 |
|
| 211 |
+
# Get actual tokenizer (handle both Tokenizer and Processor objects)
|
| 212 |
+
if hasattr(self.tokenizer, 'tokenizer'):
|
| 213 |
+
# This is a processor, extract the tokenizer
|
| 214 |
+
actual_tokenizer = self.tokenizer.tokenizer
|
| 215 |
+
else:
|
| 216 |
+
# This is already a tokenizer
|
| 217 |
+
actual_tokenizer = self.tokenizer
|
| 218 |
+
|
| 219 |
# Add special tokens to tokenizer
|
| 220 |
+
num_added_toks = actual_tokenizer.add_special_tokens(
|
| 221 |
{"additional_special_tokens": SPECIAL_TOKENS}
|
| 222 |
)
|
| 223 |
|