from pathlib import Path import sys, json, re import torch import numpy as np from collections import Counter class SimpleTokenizer: def __init__(self): self.specials = ["", "", "", ""] self.token_to_id = {tok:i for i,tok in enumerate(self.specials)} self.id_to_token = {i:tok for i,tok in enumerate(self.specials)} self.ready = False self.vocab_size = len(self.specials) def split(self, text): return re.findall(r"[A-Za-z]+|[0-9]+|\s+|[^\sA-Za-z0-9]", str(text)) def encode(self, text, add_bos=False, add_eos=False): ids=[] if add_bos: ids.append(self.token_to_id.get("",2)) unk=self.token_to_id.get("",1) for tok in self.split(text): ids.append(self.token_to_id.get(tok, unk)) if add_eos: ids.append(self.token_to_id.get("",3)) return np.array(ids,dtype=np.int64) def decode(self, ids): out=[] for idx in ids: tok=self.id_to_token.get(int(idx),"") if tok in self.specials: continue out.append(tok) return "".join(out) class Dataset: def __init__(self): self.text="" self.raw_text="" self.clean_mode="ultra" self.tokenizer=SimpleTokenizer() self.token_cache=None self.ready=False def main(): if len(sys.argv) < 3: print(r'Usage: python export_web_model.py "C:\path\save.pt" "C:\AIWeb\hf-space\web_model"') raise SystemExit(1) save_path = Path(sys.argv[1]) out_dir = Path(sys.argv[2]) out_dir.mkdir(parents=True, exist_ok=True) data = torch.load(save_path, map_location="cpu", weights_only=False) dataset = data.get("dataset") tok = dataset.tokenizer tokens = [tok.id_to_token.get(i, "") for i in range(tok.vocab_size)] config = { "vocab_size": int(tok.vocab_size), "d_model": int(data.get("d_model", 384)), "n_heads": int(data.get("n_heads", 6)), "n_layers": int(data.get("n_layers", 4)), "max_seq": int(data.get("max_seq", 512)), "dropout": 0.0, "temperature": float(data.get("temperature", 0.55)), "top_k": int(data.get("top_k", 40)), "top_p": float(data.get("top_p", 0.90)), "repeat_penalty": float(data.get("repeat_penalty", 1.25)), "no_repeat_ngram": int(data.get("no_repeat_ngram", 3)), "sample_tokens": int(data.get("sample_tokens", 180)), } model_info = { "source_save": str(save_path), "steps_done": int(data.get("steps_done", 0)), "training_chars": len(getattr(dataset, "text", "")), "vocab_size": int(tok.vocab_size), "architecture": "ScratchWebLM Token Transformer", } if data.get("state") is None: raise ValueError("Save has no model state. Train and save first.") torch.save(data["state"], out_dir / "model.pt") (out_dir / "tokenizer.json").write_text(json.dumps({"tokens": tokens}, ensure_ascii=False, indent=2), encoding="utf-8") (out_dir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8") (out_dir / "model_info.json").write_text(json.dumps(model_info, indent=2), encoding="utf-8") print("Exported to:", out_dir) for p in out_dir.iterdir(): print(p.name, p.stat().st_size, "bytes") if __name__ == "__main__": main()