| import chardet | |
| import json | |
| import os | |
| def create_pretraining_set(content_list, json_file): | |
| # Initialize a variable to store the combined text of all files | |
| # Walk through all directories and files in the directory | |
| # remove previous pretraining set if it exists | |
| if os.path.exists(json_file): | |
| os.remove(json_file) | |
| os.makedirs(os.path.dirname(json_file), exist_ok=True) | |
| for file_contents in content_list: | |
| with open(json_file, "a", encoding="utf-8", errors="ignore") as file: | |
| data = {"text": file_contents} | |
| write = json.dumps(data, ensure_ascii=False) | |
| file.write(write + "\n") | |
| # Create a dictionary with the combined text | |