File size: 700 Bytes
42da440 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | import chardet
import json
import os
def create_pretraining_set(content_list, json_file):
# Initialize a variable to store the combined text of all files
# Walk through all directories and files in the directory
# remove previous pretraining set if it exists
if os.path.exists(json_file):
os.remove(json_file)
os.makedirs(os.path.dirname(json_file), exist_ok=True)
for file_contents in content_list:
with open(json_file, "a", encoding="utf-8", errors="ignore") as file:
data = {"text": file_contents}
write = json.dumps(data, ensure_ascii=False)
file.write(write + "\n")
# Create a dictionary with the combined text
|