Download kdflow/models/utils.py from codemaivanngu/ctopd: direct link, hf CLI and curl.
- Browser
- Download file 1.76 kB
-
https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/models/utils.py
- Command line
-
hf download hf://codemaivanngu/ctopd/kdflow/models/utils.py
-
curl -L -o utils.py https://huggingface.co/codemaivanngu/ctopd/resolve/main/kdflow/models/utils.py
1.76 kB
| import logging | |
| import torch | |
| import torch.nn.functional as F | |
| from dataclasses import dataclass | |
| from typing import Union | |
| from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast | |
| logger = logging.getLogger(__name__) | |
| class TokenizerCompareResult: | |
| """Result of comparing two tokenizers.""" | |
| template_identical: bool = True | |
| vocab_identical: bool = True | |
| def is_identical(self) -> bool: | |
| """True if both chat_template and vocabulary are identical.""" | |
| return self.template_identical and self.vocab_identical | |
| def check_tokenizer_identical( | |
| tokenizer1: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], | |
| tokenizer2: Union[PreTrainedTokenizer, PreTrainedTokenizerFast], | |
| ) -> TokenizerCompareResult: | |
| """ | |
| Check if two tokenizers are identical by comparing their chat_template and vocabulary. | |
| Args: | |
| tokenizer1: The first tokenizer to compare. | |
| tokenizer2: The second tokenizer to compare. | |
| Returns: | |
| TokenizerCompareResult with template_identical and vocab_identical fields. | |
| """ | |
| result = TokenizerCompareResult() | |
| # Compare chat_template | |
| chat_template1 = getattr(tokenizer1, "chat_template", None) | |
| chat_template2 = getattr(tokenizer2, "chat_template", None) | |
| if chat_template1 != chat_template2: | |
| logger.warning("Tokenizers differ in chat_template.") | |
| result.template_identical = False | |
| # Compare vocabulary | |
| vocab1 = tokenizer1.get_vocab() | |
| vocab2 = tokenizer2.get_vocab() | |
| if vocab1 != vocab2: | |
| logger.warning( | |
| f"Tokenizers differ in vocabulary. " | |
| f"Vocab sizes: {len(vocab1)} vs {len(vocab2)}." | |
| ) | |
| result.vocab_identical = False | |
| return result | |