Kimodo_SOMA_Modular / prompt_utils.py
OzzyGT's picture
OzzyGT HF Staff
Upload 16 files
903e660 verified
Raw History Blame Contribute Delete
1.13 kB
import torch
def tokenize_prompt(tokenizer, prompt, max_length=512):
token_count = len(tokenizer(prompt, add_special_tokens=False, truncation=True, max_length=max_length)["input_ids"])
while token_count > 400:
prompt = " ".join(prompt.split()[: int(len(prompt.split()) * 400 / token_count)])
token_count = len(
tokenizer(prompt, add_special_tokens=False, truncation=True, max_length=max_length)["input_ids"]
)
formatted = "<|start_header_id|>user<|end_header_id|>\n\n" + prompt.strip() + "<|eot_id|>"
inputs = tokenizer([formatted], return_tensors="pt", padding=True, truncation=True, max_length=max_length)
# LLM2Vec excludes the instruction prefix, retaining the prompt and trailing end-of-turn token.
suffix = tokenizer(
[prompt.strip() + "<|eot_id|>"],
return_tensors="pt",
truncation=True,
max_length=max_length,
add_special_tokens=False,
)
pooling_mask = torch.zeros_like(inputs["attention_mask"])
pooling_mask[:, -suffix["input_ids"].shape[1] :] = 1
inputs["pooling_mask"] = pooling_mask
return inputs