test_dynamic_processor / tokenization_fast.py

Initial commit

24b4f96 over 4 years ago

203 Bytes

from transformers import BertTokenizerFast
from .tokenization import NewTokenizer

class NewTokenizerFast(BertTokenizerFast):
    slow_tokenizer_class = NewTokenizer
    special_attribute_present = True

	from transformers import BertTokenizerFast
	from .tokenization import NewTokenizer

	class NewTokenizerFast(BertTokenizerFast):
	slow_tokenizer_class = NewTokenizer
	special_attribute_present = True