Spaces:
Runtime error
Runtime error
File size: 3,395 Bytes
2aa5ccd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | from pathlib import Path
import sys, json, re
import torch
import numpy as np
from collections import Counter
class SimpleTokenizer:
def __init__(self):
self.specials = ["<PAD>", "<UNK>", "<BOS>", "<EOS>"]
self.token_to_id = {tok:i for i,tok in enumerate(self.specials)}
self.id_to_token = {i:tok for i,tok in enumerate(self.specials)}
self.ready = False
self.vocab_size = len(self.specials)
def split(self, text):
return re.findall(r"[A-Za-z]+|[0-9]+|\s+|[^\sA-Za-z0-9]", str(text))
def encode(self, text, add_bos=False, add_eos=False):
ids=[]
if add_bos: ids.append(self.token_to_id.get("<BOS>",2))
unk=self.token_to_id.get("<UNK>",1)
for tok in self.split(text):
ids.append(self.token_to_id.get(tok, unk))
if add_eos: ids.append(self.token_to_id.get("<EOS>",3))
return np.array(ids,dtype=np.int64)
def decode(self, ids):
out=[]
for idx in ids:
tok=self.id_to_token.get(int(idx),"")
if tok in self.specials: continue
out.append(tok)
return "".join(out)
class Dataset:
def __init__(self):
self.text=""
self.raw_text=""
self.clean_mode="ultra"
self.tokenizer=SimpleTokenizer()
self.token_cache=None
self.ready=False
def main():
if len(sys.argv) < 3:
print(r'Usage: python export_web_model.py "C:\path\save.pt" "C:\AIWeb\hf-space\web_model"')
raise SystemExit(1)
save_path = Path(sys.argv[1])
out_dir = Path(sys.argv[2])
out_dir.mkdir(parents=True, exist_ok=True)
data = torch.load(save_path, map_location="cpu", weights_only=False)
dataset = data.get("dataset")
tok = dataset.tokenizer
tokens = [tok.id_to_token.get(i, "<UNK>") for i in range(tok.vocab_size)]
config = {
"vocab_size": int(tok.vocab_size),
"d_model": int(data.get("d_model", 384)),
"n_heads": int(data.get("n_heads", 6)),
"n_layers": int(data.get("n_layers", 4)),
"max_seq": int(data.get("max_seq", 512)),
"dropout": 0.0,
"temperature": float(data.get("temperature", 0.55)),
"top_k": int(data.get("top_k", 40)),
"top_p": float(data.get("top_p", 0.90)),
"repeat_penalty": float(data.get("repeat_penalty", 1.25)),
"no_repeat_ngram": int(data.get("no_repeat_ngram", 3)),
"sample_tokens": int(data.get("sample_tokens", 180)),
}
model_info = {
"source_save": str(save_path),
"steps_done": int(data.get("steps_done", 0)),
"training_chars": len(getattr(dataset, "text", "")),
"vocab_size": int(tok.vocab_size),
"architecture": "ScratchWebLM Token Transformer",
}
if data.get("state") is None:
raise ValueError("Save has no model state. Train and save first.")
torch.save(data["state"], out_dir / "model.pt")
(out_dir / "tokenizer.json").write_text(json.dumps({"tokens": tokens}, ensure_ascii=False, indent=2), encoding="utf-8")
(out_dir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8")
(out_dir / "model_info.json").write_text(json.dumps(model_info, indent=2), encoding="utf-8")
print("Exported to:", out_dir)
for p in out_dir.iterdir():
print(p.name, p.stat().st_size, "bytes")
if __name__ == "__main__":
main()
|