from types import SimpleNamespace import pytest from runtime import validate_tokenizer class BaseTokenizer: eos_token_id = 126081 mask_token_id = None def convert_tokens_to_ids(self, text): return 126336 @property def mask_token(self): return getattr(self, "_mask_token", None) @mask_token.setter def mask_token(self, text): self._mask_token = text self.mask_token_id = self.convert_tokens_to_ids(text) def __len__(self): return 126349 def test_registers_existing_mask_token(): x = BaseTokenizer() validate_tokenizer(x, SimpleNamespace(mask_token_id=126336, eos_token_id=126081, vocab_size=126464)) assert x.mask_token_id == 126336 and len(x) == 126349 def test_rejects_incompatible_tokenizer(): with pytest.raises(RuntimeError, match="mask vocabulary"): validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=1, eos_token_id=126081, vocab_size=126464)) with pytest.raises(RuntimeError, match="special IDs"): validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=126336, eos_token_id=2, vocab_size=126464))