File size: 2,516 Bytes
921bf58 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Transformers adapter preserving literal text and the original byte BPE."""
from pathlib import Path
from transformers import PreTrainedTokenizer
from .purrence_core import Sub3MTokenizer
class PurrenceTokenizer(PreTrainedTokenizer):
vocab_files_names = {"vocab_file": "tokenizer/ordinary-tokenizer.json",
"artifact_file": "tokenizer/artifact.json"}
model_input_names = ["input_ids", "attention_mask"]
def __init__(self, vocab_file, artifact_file, **kwargs):
self.native = Sub3MTokenizer(Path(artifact_file).parent)
self.vocab_file = vocab_file
self.artifact_file = artifact_file
self.vocab = self.native.backend.get_vocab()
self.vocab["<|endoftext|>"] = 2047
self.reverse_vocab = {v: k for k, v in self.vocab.items()}
kwargs.setdefault("bos_token", "<|endoftext|>")
kwargs.setdefault("eos_token", "<|endoftext|>")
kwargs.setdefault("pad_token", "<|endoftext|>")
kwargs.setdefault("model_max_length", 8192)
kwargs["split_special_tokens"] = True
kwargs["clean_up_tokenization_spaces"] = False
super().__init__(**kwargs)
@property
def vocab_size(self):
return 2048
def get_vocab(self):
return dict(self.vocab)
def _tokenize(self, text, **kwargs):
return [self.reverse_vocab[i] for i in self.native.encode(text)]
def _convert_token_to_id(self, token):
return self.vocab[token]
def _convert_id_to_token(self, index):
return self.reverse_vocab[index]
def convert_tokens_to_string(self, tokens):
return self.native.decode([self.vocab[t] for t in tokens])
def _decode(self, token_ids, skip_special_tokens=False, **kwargs):
if isinstance(token_ids, int):
token_ids = [token_ids]
return self.native.decode([int(i) for i in token_ids], skip_boundaries=skip_special_tokens)
def save_vocabulary(self, save_directory, filename_prefix=None):
import shutil
if filename_prefix:
raise ValueError("This fixed tokenizer uses canonical filenames")
target = Path(save_directory) / "tokenizer"
target.mkdir(parents=True, exist_ok=True)
paths = []
for source in [self.vocab_file, self.artifact_file]:
dest = target / Path(source).name
if Path(source).resolve() != dest.resolve():
shutil.copyfile(source, dest)
paths.append(str(dest))
return tuple(paths)
|