File size: 654 Bytes
e153144 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.trainers import WordPieceTrainer
def train_tokenizer(files, vocab_size=50000):
tokenizer = Tokenizer(WordPiece(unk_token=":OOV:"))
tokenizer.pre_tokenizer = Whitespace()
trainer = WordPieceTrainer(
vocab_size=vocab_size,
special_tokens=[":OOV:"],
continuing_subword_prefix="##",
)
tokenizer.train(files, trainer)
return tokenizer
def save_tokenizer(tokenizer, path):
tokenizer.save(path)
def load_tokenizer(path):
return Tokenizer.from_file(path)
|