| import os |
|
|
| def process_file(file_path): |
| with open(file_path, 'r', encoding='utf-8-sig') as f: |
| text = f.read() |
| assert text |
| return text, str(file_path).split('.')[0] |
|
|
|
|
| def process_folder_recursive(folder_path): |
| all_messages = [] |
| for file in os.listdir(folder_path): |
| file_path = os.path.join(folder_path, file) |
| if os.path.isfile(file_path): |
| all_messages.append(process_file(file_path)) |
| else: |
| all_messages += process_folder_recursive(file_path) |
| return all_messages |
|
|
|
|
| def load_and_process_data() -> list[dict]: |
| """Загрузка и предобработка данных из JSON файлов""" |
| all_messages = process_folder_recursive('texts') |
| return [x[0] for x in all_messages], [x[1][:-3] for x in all_messages] |
|
|
