scratchweblm / export_web_model.py
MaTtHeBeSt4444's picture
Upload 11 files
2aa5ccd verified
Raw History Blame Contribute Delete
3.4 kB
from pathlib import Path
import sys, json, re
import torch
import numpy as np
from collections import Counter
class SimpleTokenizer:
def __init__(self):
self.specials = ["<PAD>", "<UNK>", "<BOS>", "<EOS>"]
self.token_to_id = {tok:i for i,tok in enumerate(self.specials)}
self.id_to_token = {i:tok for i,tok in enumerate(self.specials)}
self.ready = False
self.vocab_size = len(self.specials)
def split(self, text):
return re.findall(r"[A-Za-z]+|[0-9]+|\s+|[^\sA-Za-z0-9]", str(text))
def encode(self, text, add_bos=False, add_eos=False):
ids=[]
if add_bos: ids.append(self.token_to_id.get("<BOS>",2))
unk=self.token_to_id.get("<UNK>",1)
for tok in self.split(text):
ids.append(self.token_to_id.get(tok, unk))
if add_eos: ids.append(self.token_to_id.get("<EOS>",3))
return np.array(ids,dtype=np.int64)
def decode(self, ids):
out=[]
for idx in ids:
tok=self.id_to_token.get(int(idx),"")
if tok in self.specials: continue
out.append(tok)
return "".join(out)
class Dataset:
def __init__(self):
self.text=""
self.raw_text=""
self.clean_mode="ultra"
self.tokenizer=SimpleTokenizer()
self.token_cache=None
self.ready=False
def main():
if len(sys.argv) < 3:
print(r'Usage: python export_web_model.py "C:\path\save.pt" "C:\AIWeb\hf-space\web_model"')
raise SystemExit(1)
save_path = Path(sys.argv[1])
out_dir = Path(sys.argv[2])
out_dir.mkdir(parents=True, exist_ok=True)
data = torch.load(save_path, map_location="cpu", weights_only=False)
dataset = data.get("dataset")
tok = dataset.tokenizer
tokens = [tok.id_to_token.get(i, "<UNK>") for i in range(tok.vocab_size)]
config = {
"vocab_size": int(tok.vocab_size),
"d_model": int(data.get("d_model", 384)),
"n_heads": int(data.get("n_heads", 6)),
"n_layers": int(data.get("n_layers", 4)),
"max_seq": int(data.get("max_seq", 512)),
"dropout": 0.0,
"temperature": float(data.get("temperature", 0.55)),
"top_k": int(data.get("top_k", 40)),
"top_p": float(data.get("top_p", 0.90)),
"repeat_penalty": float(data.get("repeat_penalty", 1.25)),
"no_repeat_ngram": int(data.get("no_repeat_ngram", 3)),
"sample_tokens": int(data.get("sample_tokens", 180)),
}
model_info = {
"source_save": str(save_path),
"steps_done": int(data.get("steps_done", 0)),
"training_chars": len(getattr(dataset, "text", "")),
"vocab_size": int(tok.vocab_size),
"architecture": "ScratchWebLM Token Transformer",
}
if data.get("state") is None:
raise ValueError("Save has no model state. Train and save first.")
torch.save(data["state"], out_dir / "model.pt")
(out_dir / "tokenizer.json").write_text(json.dumps({"tokens": tokens}, ensure_ascii=False, indent=2), encoding="utf-8")
(out_dir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8")
(out_dir / "model_info.json").write_text(json.dumps(model_info, indent=2), encoding="utf-8")
print("Exported to:", out_dir)
for p in out_dir.iterdir():
print(p.name, p.stat().st_size, "bytes")
if __name__ == "__main__":
main()