File size: 402 Bytes
c14c4fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | from funasr.register import tables
@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
try:
from transformers import AutoTokenizer
except:
# print("If you want to use hugging, please `pip install -U transformers`")
pass
tokenizer = AutoTokenizer.from_pretrained(init_param_path)
return tokenizer
|