| |
| from ..file_utils import requires_tokenizers |
|
|
|
|
| class AlbertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class BartTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class BertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class CamembertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class DistilBertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class DPRContextEncoderTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class DPRQuestionEncoderTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class DPRReaderTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class ElectraTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class FunnelTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class GPT2TokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class HerbertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class LayoutLMTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class LongformerTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class LxmertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class MBartTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class MobileBertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class OpenAIGPTTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class PegasusTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class ReformerTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class RetriBertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class RobertaTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class SqueezeBertTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class T5TokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class PreTrainedTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class XLMRobertaTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| class XLNetTokenizerFast: |
| def __init__(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
| @classmethod |
| def from_pretrained(self, *args, **kwargs): |
| requires_tokenizers(self) |
|
|
|
|
| SLOW_TO_FAST_CONVERTERS = None |
|
|
|
|
| def convert_slow_tokenizer(*args, **kwargs): |
| requires_tokenizers(convert_slow_tokenizer) |
|
|