Download tests/test_runtime.py from User-2468/Name: direct link, hf CLI and curl.
- Browser
- Download file 1.14 kB
-
https://huggingface.co/spaces/User-2468/Name/resolve/main/tests/test_runtime.py
- Command line
-
hf download hf://spaces/User-2468/Name/tests/test_runtime.py
-
curl -L -o test_runtime.py https://huggingface.co/spaces/User-2468/Name/resolve/main/tests/test_runtime.py
1.14 kB
| from types import SimpleNamespace | |
| import pytest | |
| from runtime import validate_tokenizer | |
| class BaseTokenizer: | |
| eos_token_id = 126081 | |
| mask_token_id = None | |
| def convert_tokens_to_ids(self, text): | |
| return 126336 | |
| def mask_token(self): | |
| return getattr(self, "_mask_token", None) | |
| def mask_token(self, text): | |
| self._mask_token = text | |
| self.mask_token_id = self.convert_tokens_to_ids(text) | |
| def __len__(self): | |
| return 126349 | |
| def test_registers_existing_mask_token(): | |
| x = BaseTokenizer() | |
| validate_tokenizer(x, SimpleNamespace(mask_token_id=126336, eos_token_id=126081, vocab_size=126464)) | |
| assert x.mask_token_id == 126336 and len(x) == 126349 | |
| def test_rejects_incompatible_tokenizer(): | |
| with pytest.raises(RuntimeError, match="mask vocabulary"): | |
| validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=1, eos_token_id=126081, vocab_size=126464)) | |
| with pytest.raises(RuntimeError, match="special IDs"): | |
| validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=126336, eos_token_id=2, vocab_size=126464)) | |