File size: 3,290 Bytes
0ca529a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | import runpy
from typing import List
import torch
import torch.nn.functional as F
from transformers import AutoTokenizer
from kdflow.utils.logging_utils import init_logger
logger = init_logger(__name__)
def load_custom_eval_fn(path):
"""Load the required eval_fn from a custom Python file."""
eval_fn = runpy.run_path(path).get("eval_fn")
if not callable(eval_fn):
raise ValueError(
f"{path!r} must define a callable eval_fn(predictions, labels)"
)
return eval_fn
def get_tokenizer(model_name_or_path, model=None, padding_side="left", use_fast=True):
try:
# fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
# tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
# with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
tokenizer = AutoTokenizer.from_pretrained(
model_name_or_path,
trust_remote_code=True,
use_fast=use_fast,
fix_mistral_regex=True,
)
except TypeError:
# Older transformers versions do not accept fix_mistral_regex.
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
tokenizer.padding_side = padding_side
# NOTE: When enable vLLM, do not resize_token_embeddings, or the vocab size will mismatch with vLLM.
# https://github.com/facebookresearch/llama-recipes/pull/196
if tokenizer.pad_token is None:
logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
tokenizer.pad_token = tokenizer.eos_token
tokenizer.pad_token_id = tokenizer.eos_token_id
if model is not None:
model.config.pad_token_id = tokenizer.pad_token_id
return tokenizer
def convert_token_to_id(token, tokenizer):
if isinstance(token, str):
token = tokenizer.encode(token, add_special_tokens=False)
assert len(token) == 1
return token[0]
else:
raise ValueError("token should be int or str")
def zero_pad_sequences(
sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
) -> torch.Tensor:
from torch.nn.utils.rnn import pad_sequence
assert side in ("left", "right")
sequences = [seq.squeeze(0) for seq in sequences]
if side == "left":
sequences = [seq.flip(dims=0) for seq in sequences]
padded_sequences = pad_sequence(
sequences,
batch_first=True,
padding_value=value
)
if side == "left":
padded_sequences = torch.flip(padded_sequences, dims=[1])
return padded_sequences
def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
"""Remove the pad token. Return tensors and not lists.
Args:
input_ids shape: [bs, seq_length]
attention_mask shape: [bs, seq_length]
Returns:
no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
"""
no_padding_batch = []
for ids, mask in zip(input_ids, attention_mask):
# Fix for both left and right padding
ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
no_padding_batch.append(ids)
return no_padding_batch
|