File size: 564 Bytes
e153144 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | from tokenizer import train_tokenizer, save_tokenizer
from os import listdir
from os.path import abspath, isfile, join
if __name__ == "__main__":
print("Training tokenizer...")
files = [
abspath(join("data/rawtexts/", filename))
for filename in listdir("data/rawtexts/")
if isfile(join("data/rawtexts/", filename))
]
tokenizer = train_tokenizer(files=files)
print("Trained!")
print("Saving to trained/tokenizer_saved")
save_tokenizer(tokenizer, abspath("trained/tokenizer_saved.json"))
print("Saved!")
|