utils/train_tokenizer.py

from tokenizer import train_tokenizer, save_tokenizer
from os import listdir
from os.path import abspath, isfile, join

if __name__ == "__main__":
    print("Training tokenizer...")

    files = [
        abspath(join("data/rawtexts/", filename))
        for filename in listdir("data/rawtexts/")
        if isfile(join("data/rawtexts/", filename))
    ]

    tokenizer = train_tokenizer(files=files)

    print("Trained!")
    print("Saving to trained/tokenizer_saved")

    save_tokenizer(tokenizer, abspath("trained/tokenizer_saved.json"))

    print("Saved!")