File size: 3,395 Bytes
2aa5ccd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
from pathlib import Path
import sys, json, re
import torch
import numpy as np
from collections import Counter

class SimpleTokenizer:
    def __init__(self):
        self.specials = ["<PAD>", "<UNK>", "<BOS>", "<EOS>"]
        self.token_to_id = {tok:i for i,tok in enumerate(self.specials)}
        self.id_to_token = {i:tok for i,tok in enumerate(self.specials)}
        self.ready = False
        self.vocab_size = len(self.specials)
    def split(self, text):
        return re.findall(r"[A-Za-z]+|[0-9]+|\s+|[^\sA-Za-z0-9]", str(text))
    def encode(self, text, add_bos=False, add_eos=False):
        ids=[]
        if add_bos: ids.append(self.token_to_id.get("<BOS>",2))
        unk=self.token_to_id.get("<UNK>",1)
        for tok in self.split(text):
            ids.append(self.token_to_id.get(tok, unk))
        if add_eos: ids.append(self.token_to_id.get("<EOS>",3))
        return np.array(ids,dtype=np.int64)
    def decode(self, ids):
        out=[]
        for idx in ids:
            tok=self.id_to_token.get(int(idx),"")
            if tok in self.specials: continue
            out.append(tok)
        return "".join(out)

class Dataset:
    def __init__(self):
        self.text=""
        self.raw_text=""
        self.clean_mode="ultra"
        self.tokenizer=SimpleTokenizer()
        self.token_cache=None
        self.ready=False

def main():
    if len(sys.argv) < 3:
        print(r'Usage: python export_web_model.py "C:\path\save.pt" "C:\AIWeb\hf-space\web_model"')
        raise SystemExit(1)

    save_path = Path(sys.argv[1])
    out_dir = Path(sys.argv[2])
    out_dir.mkdir(parents=True, exist_ok=True)

    data = torch.load(save_path, map_location="cpu", weights_only=False)
    dataset = data.get("dataset")
    tok = dataset.tokenizer

    tokens = [tok.id_to_token.get(i, "<UNK>") for i in range(tok.vocab_size)]

    config = {
        "vocab_size": int(tok.vocab_size),
        "d_model": int(data.get("d_model", 384)),
        "n_heads": int(data.get("n_heads", 6)),
        "n_layers": int(data.get("n_layers", 4)),
        "max_seq": int(data.get("max_seq", 512)),
        "dropout": 0.0,
        "temperature": float(data.get("temperature", 0.55)),
        "top_k": int(data.get("top_k", 40)),
        "top_p": float(data.get("top_p", 0.90)),
        "repeat_penalty": float(data.get("repeat_penalty", 1.25)),
        "no_repeat_ngram": int(data.get("no_repeat_ngram", 3)),
        "sample_tokens": int(data.get("sample_tokens", 180)),
    }

    model_info = {
        "source_save": str(save_path),
        "steps_done": int(data.get("steps_done", 0)),
        "training_chars": len(getattr(dataset, "text", "")),
        "vocab_size": int(tok.vocab_size),
        "architecture": "ScratchWebLM Token Transformer",
    }

    if data.get("state") is None:
        raise ValueError("Save has no model state. Train and save first.")

    torch.save(data["state"], out_dir / "model.pt")
    (out_dir / "tokenizer.json").write_text(json.dumps({"tokens": tokens}, ensure_ascii=False, indent=2), encoding="utf-8")
    (out_dir / "config.json").write_text(json.dumps(config, indent=2), encoding="utf-8")
    (out_dir / "model_info.json").write_text(json.dumps(model_info, indent=2), encoding="utf-8")

    print("Exported to:", out_dir)
    for p in out_dir.iterdir():
        print(p.name, p.stat().st_size, "bytes")

if __name__ == "__main__":
    main()