File size: 1,141 Bytes
79df115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
from types import SimpleNamespace
import pytest
from runtime import validate_tokenizer

class BaseTokenizer:
    eos_token_id = 126081
    mask_token_id = None
    def convert_tokens_to_ids(self, text):
        return 126336
    @property
    def mask_token(self):
        return getattr(self, "_mask_token", None)
    @mask_token.setter
    def mask_token(self, text):
        self._mask_token = text
        self.mask_token_id = self.convert_tokens_to_ids(text)
    def __len__(self):
        return 126349

def test_registers_existing_mask_token():
    x = BaseTokenizer()
    validate_tokenizer(x, SimpleNamespace(mask_token_id=126336, eos_token_id=126081, vocab_size=126464))
    assert x.mask_token_id == 126336 and len(x) == 126349

def test_rejects_incompatible_tokenizer():
    with pytest.raises(RuntimeError, match="mask vocabulary"):
        validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=1, eos_token_id=126081, vocab_size=126464))
    with pytest.raises(RuntimeError, match="special IDs"):
        validate_tokenizer(BaseTokenizer(), SimpleNamespace(mask_token_id=126336, eos_token_id=2, vocab_size=126464))