Download kdflow/utils/utils.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 3.29 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/utils/utils.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/utils/utils.py
-
curl -L -o utils.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/utils/utils.py
3.29 kB
| import runpy | |
| from typing import List | |
| import torch | |
| import torch.nn.functional as F | |
| from transformers import AutoTokenizer | |
| from kdflow.utils.logging_utils import init_logger | |
| logger = init_logger(__name__) | |
| def load_custom_eval_fn(path): | |
| """Load the required eval_fn from a custom Python file.""" | |
| eval_fn = runpy.run_path(path).get("eval_fn") | |
| if not callable(eval_fn): | |
| raise ValueError( | |
| f"{path!r} must define a callable eval_fn(predictions, labels)" | |
| ) | |
| return eval_fn | |
| def get_tokenizer(model_name_or_path, model=None, padding_side="left", use_fast=True): | |
| try: | |
| # fix_mistral_regex keeps the ByteLevel pre-tokenizer of Tekken-style | |
| # tokenizer.json (e.g. Ministral/Mistral-Small) instead of replacing it | |
| # with the Metaspace pre-tokenizer rebuilt from the slow tokenizer. | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| model_name_or_path, | |
| trust_remote_code=True, | |
| use_fast=use_fast, | |
| fix_mistral_regex=True, | |
| ) | |
| except TypeError: | |
| # Older transformers versions do not accept fix_mistral_regex. | |
| tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True, use_fast=use_fast) | |
| tokenizer.padding_side = padding_side | |
| # NOTE: When enable vLLM, do not resize_token_embeddings, or the vocab size will mismatch with vLLM. | |
| # https://github.com/facebookresearch/llama-recipes/pull/196 | |
| if tokenizer.pad_token is None: | |
| logger.info("Detect no pad_token in tokenizer, set it to eos_token.") | |
| tokenizer.pad_token = tokenizer.eos_token | |
| tokenizer.pad_token_id = tokenizer.eos_token_id | |
| if model is not None: | |
| model.config.pad_token_id = tokenizer.pad_token_id | |
| return tokenizer | |
| def convert_token_to_id(token, tokenizer): | |
| if isinstance(token, str): | |
| token = tokenizer.encode(token, add_special_tokens=False) | |
| assert len(token) == 1 | |
| return token[0] | |
| else: | |
| raise ValueError("token should be int or str") | |
| def zero_pad_sequences( | |
| sequences: List[torch.Tensor], side: str = "left", value: int = 0, stack: bool = False | |
| ) -> torch.Tensor: | |
| from torch.nn.utils.rnn import pad_sequence | |
| assert side in ("left", "right") | |
| sequences = [seq.squeeze(0) for seq in sequences] | |
| if side == "left": | |
| sequences = [seq.flip(dims=0) for seq in sequences] | |
| padded_sequences = pad_sequence( | |
| sequences, | |
| batch_first=True, | |
| padding_value=value | |
| ) | |
| if side == "left": | |
| padded_sequences = torch.flip(padded_sequences, dims=[1]) | |
| return padded_sequences | |
| def remove_pad_token(input_ids: torch.Tensor, attention_mask: torch.Tensor, return_tensors: bool = True): | |
| """Remove the pad token. Return tensors and not lists. | |
| Args: | |
| input_ids shape: [bs, seq_length] | |
| attention_mask shape: [bs, seq_length] | |
| Returns: | |
| no_padding_batch(List[Tensor[int]]): contains the rmpad token ids per query. | |
| """ | |
| no_padding_batch = [] | |
| for ids, mask in zip(input_ids, attention_mask): | |
| # Fix for both left and right padding | |
| ids = ids[mask.bool()] if return_tensors else ids[mask.bool()].tolist() | |
| no_padding_batch.append(ids) | |
| return no_padding_batch | |