File size: 1,141 Bytes
79df115 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | from types import SimpleNamespace
import pytest
from runtime import validate_tokenizer
class BaseTokenizer:
eos_token_id = 126081
mask_token_id = None
def convert_tokens_to_ids(self, text):
return 126336
@property
def mask_token(self):
return getattr(self, "_mask_token", None)
@mask_token.setter
def mask_token(self, text):
self._mask_token = text
self.mask_token_id = self.convert_tokens_to_ids(text)
def __len__(self):
return 126349
def test_registers_existing_mask_token():
x = BaseTokenizer()
validate_tokenizer(x, SimpleNamespace(mask_token_id=126336, eos_token_id=126081, vocab_size=126464))
assert x.mask_token_id == 126336 and len(x) == 126349
def test_rejects_incompatible_tokenizer():
with pytest.raises(RuntimeError, match="mask vocabulary"):
validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=1, eos_token_id=126081, vocab_size=126464))
with pytest.raises(RuntimeError, match="special IDs"):
validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=126336, eos_token_id=2, vocab_size=126464))
|