File size: 361 Bytes
473c3a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 | from distiller.model2vec.utils import importable
importable("transformers", "tokenizer")
from distiller.model2vec.tokenizer.tokenizer import (
clean_and_create_vocabulary,
create_tokenizer,
replace_vocabulary,
turn_tokens_into_ids,
)
__all__ = ["clean_and_create_vocabulary", "create_tokenizer", "replace_vocabulary", "turn_tokens_into_ids"]
|