Spaces:
Build error
Build error
| import pytest | |
| from plastic.tokenizer.bpe import Tokenizer | |
| SAMPLE = ( | |
| "The quick brown fox jumps over the lazy dog. " | |
| "Pack my box with five dozen liquor jugs! Sphinx of black quartz, judge my vow.\n" | |
| "Numbers 12345 and symbols #$% and tabs\there and unicode café naïve façade.\n" | |
| ) * 20 | |
| def tok() -> Tokenizer: | |
| return Tokenizer.train(SAMPLE.splitlines(), vocab_size=300) | |
| def test_vocab_size_and_special_ids(tok): | |
| assert tok.vocab_size == 300 | |
| assert tok.pad_id == 0 and tok.bos_id == 1 and tok.eos_id == 2 | |
| def test_exact_roundtrip_with_whitespace(tok): | |
| text = "hello world\tfoo\nbar baz\n\n indented café" | |
| ids = tok.encode(text) | |
| assert tok.decode(ids) == text | |
| def test_bos_eos_and_skip_special(tok): | |
| ids = tok.encode("hello", add_bos=True, add_eos=True) | |
| assert ids[0] == tok.bos_id and ids[-1] == tok.eos_id | |
| assert tok.decode(ids) == "hello" | |
| assert tok.decode(ids, skip_special=False) != "hello" | |
| def test_different_words_differ(tok): | |
| assert tok.encode("fox") != tok.encode("dog") | |
| def test_save_load_identical(tok, tmp_path): | |
| path = str(tmp_path / "tokenizer.json") | |
| tok.save(path) | |
| back = Tokenizer.load(path) | |
| text = "Pack my box with five dozen liquor jugs!" | |
| assert back.encode(text) == tok.encode(text) | |
| assert back.vocab_size == tok.vocab_size | |