import os
from tokenizers import Tokenizer,models,trainers,pre_tokenizers,decoders,processors

def build_tokenizer(vocab_size=32000,files=None,save_dir="tokenizer"):
    tok=Tokenizer(models.BPE(unk_token="<unk>"))
    tok.pre_tokenizer=pre_tokenizers.ByteLevel(add_prefix_space=False)
    tok.decoder=decoders.ByteLevel()
    tok.post_processor=processors.ByteLevel(trim_offsets=False)
    specials=["<pad>","<s>","</s>","<unk>"]
    tr=trainers.BpeTrainer(vocab_size=vocab_size,min_frequency=2,
                           special_tokens=specials,show_progress=True)
    if files:
        tok.train(files,tr)
    else:
        tmp="/tmp/_meno_tok.txt"
        with open(tmp,'w') as f: f.write("Tokenizer data placeholder. "*500+"\n")
        tok.train([tmp],tr)
        os.remove(tmp)
    os.makedirs(save_dir,exist_ok=True)
    tok.save(os.path.join(save_dir,"tokenizer.json"))
    return tok

def load_tokenizer(path="tokenizer"):
    return Tokenizer.from_file(os.path.join(path,"tokenizer.json"))