renpas22 commited on
Commit ·
8268436
1
Parent(s): bb7ed44
Fix tokenize() calls to use actual tokenizer
Browse files
src/reasoning/step_level_cot.py
CHANGED
|
@@ -240,6 +240,12 @@ class StepLevelCoTTrainer:
|
|
| 240 |
|
| 241 |
def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
|
| 242 |
"""Initialize newly added token embeddings with semantic information."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 243 |
with torch.no_grad():
|
| 244 |
input_embeddings = self.model.get_input_embeddings()
|
| 245 |
|
|
@@ -249,12 +255,12 @@ class StepLevelCoTTrainer:
|
|
| 249 |
lm_head = self.model.lm_head
|
| 250 |
|
| 251 |
for i, description in enumerate(reversed(descriptions), start=1):
|
| 252 |
-
# Tokenize the description
|
| 253 |
-
tokenized =
|
| 254 |
if not tokenized:
|
| 255 |
continue
|
| 256 |
|
| 257 |
-
tokenized_ids =
|
| 258 |
|
| 259 |
# Average embeddings of description tokens
|
| 260 |
new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
|
|
@@ -266,6 +272,7 @@ class StepLevelCoTTrainer:
|
|
| 266 |
lm_head.weight[-i, :] = new_lm_head.clone().detach()
|
| 267 |
|
| 268 |
logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
|
|
|
|
| 269 |
|
| 270 |
def load_step_dataset(
|
| 271 |
self,
|
|
|
|
| 240 |
|
| 241 |
def _initialize_new_token_embeddings(self, num_added_toks: int, descriptions: list) -> None:
|
| 242 |
"""Initialize newly added token embeddings with semantic information."""
|
| 243 |
+
# Get actual tokenizer (handle both Tokenizer and Processor objects)
|
| 244 |
+
if hasattr(self.tokenizer, 'tokenizer'):
|
| 245 |
+
actual_tokenizer = self.tokenizer.tokenizer
|
| 246 |
+
else:
|
| 247 |
+
actual_tokenizer = self.tokenizer
|
| 248 |
+
|
| 249 |
with torch.no_grad():
|
| 250 |
input_embeddings = self.model.get_input_embeddings()
|
| 251 |
|
|
|
|
| 255 |
lm_head = self.model.lm_head
|
| 256 |
|
| 257 |
for i, description in enumerate(reversed(descriptions), start=1):
|
| 258 |
+
# Tokenize the description using actual tokenizer
|
| 259 |
+
tokenized = actual_tokenizer.tokenize(description)
|
| 260 |
if not tokenized:
|
| 261 |
continue
|
| 262 |
|
| 263 |
+
tokenized_ids = actual_tokenizer.convert_tokens_to_ids(tokenized)
|
| 264 |
|
| 265 |
# Average embeddings of description tokens
|
| 266 |
new_embedding = input_embeddings.weight[tokenized_ids].mean(dim=0)
|
|
|
|
| 272 |
lm_head.weight[-i, :] = new_lm_head.clone().detach()
|
| 273 |
|
| 274 |
logger.info(f"Initialized {num_added_toks} new token embeddings with semantic information.")
|
| 275 |
+
|
| 276 |
|
| 277 |
def load_step_dataset(
|
| 278 |
self,
|