Spaces:
Runtime error
Runtime error
Download export_web_model.py from AI121212121/scratchweblm: direct link, hf CLI and curl.
- Browser
- Download file 3.4 kB
-
https://huggingface.co/spaces/AI121212121/scratchweblm/resolve/main/export_web_model.py
- Command line
-
hf download hf://spaces/AI121212121/scratchweblm/export_web_model.py
-
curl -L -o export_web_model.py https://huggingface.co/spaces/AI121212121/scratchweblm/resolve/main/export_web_model.py
3.4 kB
| from pathlib import Path | |
| import sys, json, re | |
| import torch | |
| import numpy as np | |
| from collections import Counter | |
| class SimpleTokenizer: | |
| def __init__(self): | |
| self.specials = ["<PAD>", "<UNK>", "<BOS>", "<EOS>"] | |
| self.token_to_id = {tok:i for i,tok in enumerate(self.specials)} | |
| self.id_to_token = {i:tok for i,tok in enumerate(self.specials)} | |
| self.ready = False | |
| self.vocab_size = len(self.specials) | |
| def split(self, text): | |
| return re.findall(r"[A-Za-z]+|[0-9]+|\s+|[^\sA-Za-z0-9]", str(text)) | |
| def encode(self, text, add_bos=False, add_eos=False): | |
| ids=[] | |
| if add_bos: ids.append(self.token_to_id.get("<BOS>",2)) | |
| unk=self.token_to_id.get("<UNK>",1) | |
| for tok in self.split(text): | |
| ids.append(self.token_to_id.get(tok, unk)) | |
| if add_eos: ids.append(self.token_to_id.get("<EOS>",3)) | |
| return np.array(ids,dtype=np.int64) | |
| def decode(self, ids): | |
| out=[] | |
| for idx in ids: | |
| tok=self.id_to_token.get(int(idx),"") | |
| if tok in self.specials: continue | |
| out.append(tok) | |
| return "".join(out) | |
| class Dataset: | |
| def __init__(self): | |
| self.text="" | |
| self.raw_text="" | |
| self.clean_mode="ultra" | |
| self.tokenizer=SimpleTokenizer() | |
| self.token_cache=None | |
| self.ready=False | |
| def main(): | |
| if len(sys.argv) < 3: | |
| print(r'Usage: python export_web_model.py "C:\path\save.pt" "C:\AIWeb\hf-space\web_model"') | |
| raise SystemExit(1) | |
| save_path = Path(sys.argv[1]) | |
| out_dir = Path(sys.argv[2]) | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| data = torch.load(save_path, map_location="cpu", weights_only=False) | |
| dataset = data.get("dataset") | |
| tok = dataset.tokenizer | |
| tokens = [tok.id_to_token.get(i, "<UNK>") for i in range(tok.vocab_size)] | |
| config = { | |
| "vocab_size": int(tok.vocab_size), | |
| "d_model": int(data.get("d_model", 384)), | |
| "n_heads": int(data.get("n_heads", 6)), | |
| "n_layers": int(data.get("n_layers", 4)), | |
| "max_seq": int(data.get("max_seq", 512)), | |
| "dropout": 0.0, | |
| "temperature": float(data.get("temperature", 0.55)), | |
| "top_k": int(data.get("top_k", 40)), | |
| "top_p": float(data.get("top_p", 0.90)), | |
| "repeat_penalty": float(data.get("repeat_penalty", 1.25)), | |
| "no_repeat_ngram": int(data.get("no_repeat_ngram", 3)), | |
| "sample_tokens": int(data.get("sample_tokens", 180)), | |
| } | |
| model_info = { | |
| "source_save": str(save_path), | |
| "steps_done": int(data.get("steps_done", 0)), | |
| "training_chars": len(getattr(dataset, "text", "")), | |
| "vocab_size": int(tok.vocab_size), | |
| "architecture": "ScratchWebLM Token Transformer", | |
| } | |
| if data.get("state") is None: | |
| raise ValueError("Save has no model state. Train and save first.") | |
| torch.save(data["state"], out_dir / "model.pt") | |
| (out_dir / "tokenizer.json").write_text(json.dumps({"tokens": tokens}, ensure_ascii=False, indent=2), encoding="utf-8") | |
| (out_dir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8") | |
| (out_dir / "model_info.json").write_text(json.dumps(model_info, indent=2), encoding="utf-8") | |
| print("Exported to:", out_dir) | |
| for p in out_dir.iterdir(): | |
| print(p.name, p.stat().st_size, "bytes") | |
| if __name__ == "__main__": | |
| main() | |