| from huggingface_hub import from_pretrained_keras |
| import tensorflow as tf |
| import gradio as gr |
| import nltk |
| import json |
| nltk.download('brown') |
| from nltk.corpus import brown |
| from nltk import word_tokenize |
| nltk.download('punkt') |
| nltk.download('stopwords') |
| nltk.download('wordnet') |
| from nltk.corpus import stopwords |
| from nltk import pos_tag |
| nltk.download('averaged_perceptron_tagger') |
| import re |
| import torch |
| from transformers import BertTokenizer, BertModel, BertForMaskedLM |
| from wordfreq import zipf_frequency |
| import keras |
| from keras_preprocessing.sequence import pad_sequences |
| from tensorflow.keras.utils import to_categorical |
| from huggingface_hub import hf_hub_download |
| import numpy as np |
| sent_max_length = 103 |
|
|
| bert_model = 'bert-large-uncased' |
| tokenizer = BertTokenizer.from_pretrained(bert_model) |
| model = BertForMaskedLM.from_pretrained(bert_model) |
|
|
| model_cwi = from_pretrained_keras("jaimin/CWI") |
|
|
| stop_words_ = set(stopwords.words('english')) |
|
|
|
|
| with open(hf_hub_download(repo_id="jaimin/CWI", filename="word2index.json")) as outfile: |
| word2index = json.load(outfile) |
| with open(hf_hub_download(repo_id="jaimin/CWI", filename="index2word.json")) as indexfile: |
| index2word = json.load(indexfile) |
|
|
| def cleaner(word): |
| |
| word = re.sub(r'((http|https)\:\/\/)?[a-zA-Z0-9\.\/\?\:@\-_=#]+\.([a-zA-Z]){2,6}([a-zA-Z0-9\.\&\/\?\:@\-_=#])*', |
| '', word, flags=re.MULTILINE) |
| word = re.sub('[\W]', ' ', word) |
| word = re.sub('[^a-zA-Z]', ' ', word) |
| return word.lower().strip() |
| |
| def process_input(input_text): |
| input_text = cleaner(input_text) |
| clean_text = [] |
| index_list =[] |
| input_token = [] |
| index_list_zipf = [] |
| for i, word in enumerate(input_text.split()): |
| if word in word2index: |
| clean_text.append(word) |
| input_token.append(word2index[word]) |
| else: |
| index_list.append(i) |
| input_padded = pad_sequences(maxlen=sent_max_length, sequences=[input_token], padding="post", value=0) |
| return input_padded, index_list, len(clean_text) |
| |
| def complete_missing_word(pred_binary, index_list, len_list): |
| list_cwi_predictions = list(pred_binary[0][:len_list]) |
| for i in index_list: |
| list_cwi_predictions.insert(i, 0) |
| return list_cwi_predictions |
| |
|
|
| def get_bert_candidates(input_text, list_cwi_predictions, numb_predictions_displayed = 10): |
| list_candidates_bert = [] |
| for word,pred in zip(input_text.split(), list_cwi_predictions): |
| if (pred and (pos_tag([word])[0][1] in ['NNS', 'NN', 'VBP', 'RB', 'VBG','VBD' ])) or (zipf_frequency(word, 'en')) <3.1: |
| replace_word_mask = input_text.replace(word, '[MASK]') |
| text = f'[CLS]{replace_word_mask} [SEP] {input_text} [SEP] ' |
| tokenized_text = tokenizer.tokenize(text) |
| masked_index = [i for i, x in enumerate(tokenized_text) if x == '[MASK]'][0] |
| indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) |
| segments_ids = [0]*len(tokenized_text) |
| tokens_tensor = torch.tensor([indexed_tokens]) |
| segments_tensors = torch.tensor([segments_ids]) |
| |
| with torch.no_grad(): |
| outputs = model(tokens_tensor, token_type_ids=segments_tensors) |
| predictions = outputs[0][0][masked_index] |
| predicted_ids = torch.argsort(predictions, descending=True)[:numb_predictions_displayed] |
| predicted_tokens = tokenizer.convert_ids_to_tokens(list(predicted_ids)) |
| list_candidates_bert.append((word, predicted_tokens)) |
| return list_candidates_bert |
| |
| def cwi(input_text): |
| new_text = input_text |
| input_padded, index_list, len_list = process_input(input_text) |
| pred_cwi = model_cwi.predict(input_padded) |
| pred_cwi_binary = np.argmax(pred_cwi, axis = 2) |
| complete_cwi_predictions = complete_missing_word(pred_cwi_binary, index_list, len_list) |
| bert_candidates = get_bert_candidates(input_text, complete_cwi_predictions) |
| for word_to_replace, l_candidates in bert_candidates: |
| tuples_word_zipf = [] |
| for w in l_candidates: |
| if w.isalpha(): |
| tuples_word_zipf.append((w, zipf_frequency(w, 'en'))) |
| tuples_word_zipf = sorted(tuples_word_zipf, key = lambda x: x[1], reverse=True) |
| new_text = re.sub(word_to_replace, tuples_word_zipf[0][0], new_text) |
| return new_text |
| |
| interface = gr.Interface(fn=cwi, |
| inputs=["text"], |
| outputs="text", |
| title='CWI') |
| |
|
|
| interface.launch(inline=False) |
