File size: 1,755 Bytes
0ca529a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | import logging
import torch
import torch.nn.functional as F
from dataclasses import dataclass
from typing import Union
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
logger = logging.getLogger(__name__)
@dataclass
class TokenizerCompareResult:
"""Result of comparing two tokenizers."""
template_identical: bool = True
vocab_identical: bool = True
@property
def is_identical(self) -> bool:
"""True if both chat_template and vocabulary are identical."""
return self.template_identical and self.vocab_identical
def check_tokenizer_identical(
tokenizer1: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
tokenizer2: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
) -> TokenizerCompareResult:
"""
Check if two tokenizers are identical by comparing their chat_template and vocabulary.
Args:
tokenizer1: The first tokenizer to compare.
tokenizer2: The second tokenizer to compare.
Returns:
TokenizerCompareResult with template_identical and vocab_identical fields.
"""
result = TokenizerCompareResult()
# Compare chat_template
chat_template1 = getattr(tokenizer1, "chat_template", None)
chat_template2 = getattr(tokenizer2, "chat_template", None)
if chat_template1 != chat_template2:
logger.warning("Tokenizers differ in chat_template.")
result.template_identical = False
# Compare vocabulary
vocab1 = tokenizer1.get_vocab()
vocab2 = tokenizer2.get_vocab()
if vocab1 != vocab2:
logger.warning(
f"Tokenizers differ in vocabulary. "
f"Vocab sizes: {len(vocab1)} vs {len(vocab2)}."
)
result.vocab_identical = False
return result
|