toolbox/torch/utils/data/tokenizers/tokenizer.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
from typing import List


class Tokenizer(object):
    def tokenize(self, text: str) -> List[str]:
        raise NotImplementedError


if __name__ == '__main__':
    pass