| import pickle |
| from tqdm import tqdm |
| from collections import defaultdict |
| from utils import CONFIG, save_dataset, save_dictionary_items |
|
|
|
|
| MIN_WORD_SIZE = CONFIG['min_word_size'] |
| PREFIX_FILTER_LENGTH = CONFIG['prefix_filter_length'] |
| VECT_PATH = CONFIG['vect_words_path'] |
| CLS_CLASSES_PATH = CONFIG['cls_classes_path'] |
| DICT_WORDS_PATH = CONFIG['dics_path'] |
|
|
|
|
| def generate(vec_words, main_cls_dic): |
| dict_words = [] |
| rez_dict = defaultdict(list) |
| for word in tqdm(vec_words, desc="Generating lemma dataset"): |
| dic = vec_words[word] |
| x_vec = dic['vect'] |
|
|
| for form in dic['forms']: |
| main_cls = main_cls_dic[form['main']] |
|
|
| if 'lemma' in form: |
| word_y = form['lemma'] |
| else: |
| continue |
|
|
| if word_y not in vec_words \ |
| or MIN_WORD_SIZE > len(word_y) \ |
| or MIN_WORD_SIZE > len(word): |
| continue |
|
|
| if word_y[:PREFIX_FILTER_LENGTH] != word[:PREFIX_FILTER_LENGTH] \ |
| and word_y[:PREFIX_FILTER_LENGTH].replace('ё', 'е') != word[:PREFIX_FILTER_LENGTH].replace('ё', 'е')\ |
| and form['post'] != 'comp': |
| |
| dict_words.append(dict( |
| text=word, |
| text_y=word_y, |
| main=main_cls, |
| id=form['inflect_id'] |
| )) |
| continue |
|
|
| y_vec = vec_words[word_y]['vect'] |
| items = rez_dict[main_cls] |
| items.append({ |
| 'id': form['inflect_id'], |
| 'x_src': word, |
| 'x': x_vec[0], |
| 'x_len': x_vec[1], |
| 'y_src': word_y, |
| 'y': y_vec[0], |
| 'y_len': y_vec[1], |
| 'main_cls': main_cls |
| }) |
| rez_dict[main_cls] = items |
|
|
| save_dataset(rez_dict, 'lemma') |
| save_dictionary_items(dict_words, 'lemma') |
|
|
|
|
| with open(VECT_PATH, 'rb') as f: |
| vwords = pickle.load(f) |
|
|
| with open(CLS_CLASSES_PATH, 'rb') as f: |
| cls_dic = pickle.load(f) |
|
|
| generate(vwords, cls_dic) |
|
|
