import torch def tokenize_prompt(tokenizer, prompt, max_length=512): token_count = len(tokenizer(prompt, add_special_tokens=False, truncation=True, max_length=max_length)["input_ids"]) while token_count > 400: prompt = " ".join(prompt.split()[: int(len(prompt.split()) * 400 / token_count)]) token_count = len( tokenizer(prompt, add_special_tokens=False, truncation=True, max_length=max_length)["input_ids"] ) formatted = "<|start_header_id|>user<|end_header_id|>\n\n" + prompt.strip() + "<|eot_id|>" inputs = tokenizer([formatted], return_tensors="pt", padding=True, truncation=True, max_length=max_length) # LLM2Vec excludes the instruction prefix, retaining the prompt and trailing end-of-turn token. suffix = tokenizer( [prompt.strip() + "<|eot_id|>"], return_tensors="pt", truncation=True, max_length=max_length, add_special_tokens=False, ) pooling_mask = torch.zeros_like(inputs["attention_mask"]) pooling_mask[:, -suffix["input_ids"].shape[1] :] = 1 inputs["pooling_mask"] = pooling_mask return inputs