| """Contains tokenizers like GloveTokenizers and BERT Tokenizer.""" |
|
|
| import torch |
| |
| |
| from src.utils.mapper import configmapper |
| from transformers import AutoTokenizer |
|
|
|
|
| class Tokenizer: |
| """Abstract Class for Tokenizers.""" |
|
|
| def tokenize(self): |
| """Abstract Method for tokenization.""" |
|
|
|
|
| @configmapper.map("tokenizers", "glove") |
| class GloveTokenizer(Tokenizer): |
| """Implement GloveTokenizer for tokenizing text for Glove Embeddings. |
| |
| Attributes: |
| embeddings (torchtext.vocab.Vectors): Loaded pre-trained embeddings. |
| text_field (torchtext.data.Field): Text_field for vector creation. |
| |
| Methods: |
| __init__(self, name='840B', dim='300', cache='../embeddings/') : Constructor method |
| initialize_vectors(fix_length=4, tokenize='spacy', file_path="../data/imperceptibility |
| /Concreteness Ratings/train/forty.csv", |
| file_format='tsv', fields=None): Initialize vocab vectors based on data. |
| |
| tokenize(x_input, **initializer_params): Tokenize given input and return the output. |
| """ |
|
|
| def __init__(self, name="840B", dim="300", cache="../embeddings/"): |
| """Construct GloveTokenizer. |
| |
| Args: |
| name (str): Name of the GloVe embedding file |
| dim (str): Dimensions of the Glove embedding file |
| cache (str): Path to the embeddings directory |
| """ |
| super(GloveTokenizer, self).__init__() |
| self.embeddings = GloVe(name=name, dim=dim, cache=cache) |
| self.text_field = None |
|
|
| def initialize_vectors( |
| self, |
| fix_length=4, |
| tokenize="spacy", |
| tokenizer_file_paths=None, |
| file_format="tsv", |
| fields=None, |
| ): |
| """Initialize words/sequences based on GloVe embedding. |
| |
| Args: |
| fields (list): The list containing the fields to be taken |
| and processed from the file (see documentation for |
| torchtext.data.TabularDataset) |
| fix_length (int): The length of the tokenized text, |
| padding or cropping is done accordingly |
| tokenize (function or string): Method to tokenize the data. |
| If 'spacy' uses spacy tokenizer, |
| else the specified method. |
| tokenizer_file_paths (list of str): The paths of the files containing the data |
| format (str): The format of the file : 'csv', 'tsv' or 'json' |
| """ |
| text_field = Field(batch_first=True, fix_length=fix_length, tokenize=tokenize) |
| tab_dats = [ |
| TabularDataset( |
| i, format=file_format, fields={k: (k, text_field) for k in fields} |
| ) |
| for i in tokenizer_file_paths |
| ] |
| text_field.build_vocab(*tab_dats) |
| text_field.vocab.load_vectors(self.embeddings) |
| self.text_field = text_field |
|
|
| def tokenize(self, x_input, **init_vector__params): |
| """Tokenize given input based on initialized vectors. |
| |
| Initialize the vectors with given parameters if not already initialized. |
| |
| Args: |
| x_input (str): Unprocessed input text to be tokenized |
| **initializer_params (Keyword arguments): Parameters to initialize vectors |
| |
| Returns: |
| x_output (str): Processed and tokenized text |
| """ |
| if self.text_field is None: |
| self.initialize_vectors(**init_vector__params) |
| try: |
| x_output = torch.squeeze( |
| self.text_field.process([self.text_field.preprocess(x_input)]) |
| ) |
| except Exception as e: |
| print(x_input) |
| print(self.text_field.preprocess(x_input)) |
| print(e) |
| return x_output |
|
|
|
|
| @configmapper.map("tokenizers", "AutoTokenizer") |
| class AutoTokenizer(AutoTokenizer): |
| def __init__(self, *args): |
| super(AutoTokenizer, self).__init__() |
|
|
