File size: 3,290 Bytes
0ca529a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
import runpy
from typing import List

import torch
import torch.nn.functional as F
from transformers import AutoTokenizer

from kdflow.utils.logging_utils import init_logger

logger = init_logger(__name__)


def load_custom_eval_fn(path):
    """Load the required eval_fn from a custom Python file."""
    eval_fn = runpy.run_path(path).get("eval_fn")
    if not callable(eval_fn):
        raise ValueError(
            f"{path!r} must define a callable eval_fn(predictions, labels)"
        )
    return eval_fn


def get_tokenizer(model_name_or_path, model=None, padding_side="left", use_fast=True):
    try:
        # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style
        # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it
        # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer.
        tokenizer = AutoTokenizer.from_pretrained(
            model_name_or_path,
            trust_remote_code=True,
            use_fast=use_fast,
            fix_mistral_regex=True,
        )
    except TypeError:
        # Older transformers versions do not accept fix_mistral_regex.
        tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast)
    tokenizer.padding_side = padding_side
    # NOTE: When enable vLLM, do not resize_token_embeddings, or the vocab size will mismatch with vLLM.
    # https://github.com/facebookresearch/llama-recipes/pull/196
    if tokenizer.pad_token is None:
        logger.info("Detect no pad_token in tokenizer, set it to eos_token.")
        tokenizer.pad_token = tokenizer.eos_token
        tokenizer.pad_token_id = tokenizer.eos_token_id
        if model is not None:
            model.config.pad_token_id = tokenizer.pad_token_id

    return tokenizer


def convert_token_to_id(token, tokenizer):
    if isinstance(token, str):
        token = tokenizer.encode(token, add_special_tokens=False)
        assert len(token) == 1
        return token[0]
    else:
        raise ValueError("token should be int or str")


def zero_pad_sequences(
    sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False
) -> torch.Tensor:
    from torch.nn.utils.rnn import pad_sequence
    assert side in ("left", "right")
    sequences = [seq.squeeze(0) for seq in sequences]
    if side == "left":
        sequences = [seq.flip(dims=0) for seq in sequences]
    padded_sequences = pad_sequence(
        sequences,
        batch_first=True,
        padding_value=value
    )
    if side == "left":
        padded_sequences = torch.flip(padded_sequences, dims=[1])
    return padded_sequences


def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True):
    """Remove the pad token. Return tensors and not lists.

    Args:
        input_ids shape: [bs, seq_length]
        attention_mask shape: [bs, seq_length]
    Returns:
        no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query.
    """
    no_padding_batch = []
    for ids, mask in zip(input_ids, attention_mask):
        # Fix for both left and right padding
        ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist()
        no_padding_batch.append(ids)
    return no_padding_batch