File size: 10,950 Bytes
2394b34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3f9a8bb
ab3bf61
2394b34
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0f6d08e
 
031335e
 
0f6d08e
4d2f6f6
2394b34
4d2f6f6
 
 
 
 
 
d9a2eee
2394b34
 
031335e
0f6d08e
d9a2eee
2394b34
 
0f6d08e
 
 
031335e
0f6d08e
031335e
 
 
 
 
2394b34
031335e
 
 
2394b34
26db9f2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
"""

Treevik-1 500M Instruct β€” HuggingFace Space

Downloads model from HF Hub: PlusBIntelligenceLab/Treevik1-500M-Instruct

"""

import os
import json
import pickle
from dataclasses import dataclass

import torch
import torch.nn as nn
import torch.nn.functional as F
import gradio as gr
from huggingface_hub import hf_hub_download



# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Download Model from HuggingFace Hub                                     β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•

REPO_ID = "PlusBIntelligenceLab/Treevik1-500M-Instruct"

print(f"Downloading model from {REPO_ID}...")
meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
weights_path = hf_hub_download(REPO_ID, "treevik1-500M-Instruct.pt")
tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
print("βœ… Downloaded!")

# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Model Architecture                                                      β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

def has_ve(li, nl):
    return li % 2 == (nl - 1) % 2

def apply_rotary_emb(x, cos, sin):
    d = x.shape[3] // 2
    x1, x2 = x[..., :d], x[..., d:]
    return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3)

class CausalSelfAttention(nn.Module):
    def __init__(self, config, layer_idx):
        super().__init__()
        self.n_head, self.n_kv_head, self.n_embd = config.n_head, config.n_kv_head, config.n_embd
        self.head_dim = self.n_embd // self.n_head
        self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
        self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
        self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
        self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
        self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None

    def forward(self, x, ve, cos_sin, window_size):
        B, T, C = x.size()
        q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
        k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
        v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
        if ve is not None:
            ve = ve.view(B, T, self.n_kv_head, self.head_dim)
            v = v + (2 * torch.sigmoid(self.ve_gate(x[..., :32]))).unsqueeze(-1) * ve
        cos, sin = cos_sin
        q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
        q, k = norm(q), norm(k)
        nr = self.n_head // self.n_kv_head
        if nr > 1:
            k = k.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
            v = v.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
        q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
        wl = window_size[0]
        if 0 < wl < T:
            ri, ci = torch.arange(T, device=x.device).unsqueeze(1), torch.arange(T, device=x.device).unsqueeze(0)
            y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
        else:
            y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))

class MLP(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
        self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
    def forward(self, x):
        return self.c_proj(F.relu(self.c_fc(x)).square())

class Block(nn.Module):
    def __init__(self, config, li):
        super().__init__()
        self.attn = CausalSelfAttention(config, li)
        self.mlp = MLP(config)
    def forward(self, x, ve, cs, ws):
        x = x + self.attn(norm(x), ve, cs, ws)
        x = x + self.mlp(norm(x))
        return x

@dataclass
class GPTConfig:
    sequence_len: int = 2048; vocab_size: int = 65536; n_layer: int = 12
    n_head: int = 6; n_kv_head: int = 6; n_embd: int = 768; window_pattern: str = "SSSL"

class GPT(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        pat = config.window_pattern.upper()
        lw, sw = config.sequence_len, config.sequence_len // 2
        self.window_sizes = [((sw, 0) if c == 'S' else (lw, 0)) for i, c in enumerate((pat * 4)[:config.n_layer])]
        self.window_sizes[-1] = (lw, 0)
        pad = ((config.vocab_size + 63) // 64) * 64
        self.transformer = nn.ModuleDict({"wte": nn.Embedding(pad, config.n_embd), "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)])})
        self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
        self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))
        self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))
        hd = config.n_embd // config.n_head
        self.value_embeds = nn.ModuleDict({str(i): nn.Embedding(pad, config.n_kv_head * hd) for i in range(config.n_layer) if has_ve(i, config.n_layer)})
        self.rotary_seq_len = config.sequence_len * 10
        inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd))
        freqs = torch.outer(torch.arange(self.rotary_seq_len, dtype=torch.float32), inv)
        self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
        self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)

    def forward(self, idx):
        B, T = idx.size()
        cs = self.cos[:, :T], self.sin[:, :T]
        x = norm(self.transformer.wte(idx)); x0 = x
        for i, b in enumerate(self.transformer.h):
            x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
            ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
            x = b(x, ve, cs, self.window_sizes[i])
        logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
        return 15 * torch.tanh(logits / 15)

    @torch.inference_mode()
    def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
        dev = next(self.parameters()).device
        ids = torch.tensor([tokens], dtype=torch.long, device=dev)
        for _ in range(max_tokens):
            logits = self.forward(ids[:, -self.config.sequence_len:])[:, -1, :]
            if top_k > 0:
                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
                logits[logits < v[:, [-1]]] = -float('Inf')
            if temperature > 0:
                nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
            else:
                nid = torch.argmax(logits, dim=-1, keepdim=True)
            ids = torch.cat((ids, nid), dim=1)
            yield nid.item()

# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Load Model                                                              β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•

meta = json.load(open(meta_path))
mc = meta["model_config"]
config = GPTConfig(**{k: mc[k] for k in ['sequence_len','vocab_size','n_layer','n_head','n_kv_head','n_embd']})
config.window_pattern = mc.get("window_pattern", "SSSL")

device = "cuda" if torch.cuda.is_available() else "cpu"
model = GPT(config)
model.load_state_dict(torch.load(weights_path, map_location="cpu", weights_only=True))
model = model.to(device=device, dtype=torch.float32).eval()

if device == "cuda":
    hd = config.n_embd // config.n_head
    inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32, device=device) / hd))
    freqs = torch.outer(torch.arange(model.rotary_seq_len, dtype=torch.float32, device=device), inv)
    model.cos = freqs.cos()[None, :, None, :]
    model.sin = freqs.sin()[None, :, None, :]

with open(tokenizer_path, "rb") as f:
    enc = pickle.load(f)

BOS = enc.encode_single_token("<|bos|>")
USER_START = enc.encode_single_token("<|user_start|>")
USER_END = enc.encode_single_token("<|user_end|>")
ASST_START = enc.encode_single_token("<|assistant_start|>")
ASST_END = enc.encode_single_token("<|assistant_end|>")

print(f"βœ… Loaded on {device} ({sum(p.numel() for p in model.parameters())/1e6:.0f}M params)")

def respond(message, chat_history):
    if not message.strip():
        yield ""
        return

    # Build prompt from messages-format history
    ids = [BOS]
    for entry in chat_history:
        role, content = entry["role"], entry["content"]
        if role == "user":
            ids += [USER_START] + enc.encode_ordinary(content) + [USER_END]
        elif role == "assistant":
            ids += [ASST_START] + enc.encode_ordinary(content) + [ASST_END]
    ids += [USER_START] + enc.encode_ordinary(message) + [USER_END]
    ids.append(ASST_START)

    # Generate full response
    generated = []
    for token in model.generate(ids, max_tokens=512, temperature=0.7, top_k=50):
        if token == ASST_END:
            break
        generated.append(token)

    response = enc.decode(generated)
    yield response

demo = gr.ChatInterface(
    respond,
    type="messages",
    title="🌳 Treevik-1 500M Instruct",
    description="**By PlusB Intelligence Lab** β€’ Created by Sayan Chowdhury\n\n*500M parameter model β€” may produce incorrect information.*",
    theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue"),
    chatbot=gr.Chatbot(height=500),
    textbox=gr.Textbox(placeholder="Type a message and press Enter...", container=False, scale=7),
)

demo.launch(server_name="0.0.0.0", server_port=7860)