File size: 1,471 Bytes
e153144 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | from tokenizer import load_tokenizer
from os import listdir
from tqdm import tqdm
import subprocess
if __name__ == "__main__":
print("Preparing corpus for training...")
tokenizer = load_tokenizer("trained/tokenizer_saved.json")
print("Save path: data/pretraining/corpus_tokenized.txt")
with open(
"data/pretraining/corpus_tokenized.txt",
"w",
encoding="utf-8",
) as corpus_f:
for filename in listdir("data/rawtexts/"):
with open(
"data/rawtexts/" + filename,
"r",
encoding="utf-8",
) as f:
line = f.readline()
with tqdm(
desc=f"Preprocessing corpus file: {filename}",
unit="lines",
total=int(
subprocess.run(
["wc", "-l", "data/rawtexts/" + str(filename)],
capture_output=True,
text=True,
).stdout.split()[0]
),
) as prog:
while line != "":
encoding = tokenizer.encode(line)
for token_id in encoding.ids:
corpus_f.write(str(token_id))
corpus_f.write(" ")
line = f.readline()
prog.update(1)
print("Done!")
|