PlusBIntelligenceLab commited on
Commit
8800113
Β·
verified Β·
1 Parent(s): 8f1b69d

Chat- for treevik-2

Browse files
Files changed (3) hide show
  1. README.md +34 -12
  2. app.py +266 -0
  3. requirements.txt +4 -0
README.md CHANGED
@@ -1,12 +1,34 @@
1
- ---
2
- title: Treevik2 Chat
3
- emoji: ⚑
4
- colorFrom: purple
5
- colorTo: blue
6
- sdk: gradio
7
- sdk_version: 6.6.0
8
- app_file: app.py
9
- pinned: false
10
- ---
11
-
12
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Treevik-2 2.5B Instruct
3
+ emoji: 🌳
4
+ colorFrom: indigo
5
+ colorTo: blue
6
+ sdk: gradio
7
+ sdk_version: 5.12.0
8
+ python_version: 3.12.0
9
+ app_file: app.py
10
+ pinned: true
11
+ license: apache-2.0
12
+ ---
13
+
14
+ # 🌳 Treevik-2 2.5B Instruct
15
+
16
+ The second language model in the **Treevik** family by **PlusB Intelligence Lab**, fine-tuned for conversation.
17
+
18
+ - **Parameters:** 2.5B (8-bit quantized)
19
+ - **Architecture:** GPT-style with RoPE, GQA, ReLUΒ², Value Embeddings
20
+ - **Training:** SFT on OpenHermes 2.5 (1M conversations) + identity data
21
+ - **Quantization:** 8-bit weight quantization for efficient inference
22
+ - **Creator:** Sayan Chowdhury, PlusB Intelligence Lab
23
+
24
+ ## Model Details
25
+
26
+ | Config | Value |
27
+ |--------|-------|
28
+ | Layers | 26 |
29
+ | Heads | 13 |
30
+ | KV Heads | 13 |
31
+ | Embedding Dim | 1664 |
32
+ | Sequence Length | 2048 |
33
+ | Vocab Size | 65536 |
34
+ | Window Pattern | L (full context) |
app.py ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Treevik-2 2.5B Instruct (8-bit Quantized) β€” HuggingFace Space
3
+ Downloads model from HF Hub: PlusBIntelligenceLab/Treevik2-2.5B-Instruct
4
+ """
5
+
6
+ import os
7
+ import json
8
+ import pickle
9
+ from dataclasses import dataclass
10
+
11
+ import torch
12
+ import torch.nn as nn
13
+ import torch.nn.functional as F
14
+ import gradio as gr
15
+ from huggingface_hub import hf_hub_download
16
+
17
+
18
+
19
+ # ╔══════════════════════════════════════════════════════════════════════════╗
20
+ # β•‘ Download Model from HuggingFace Hub β•‘
21
+ # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
22
+
23
+ REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
24
+
25
+ print(f"Downloading model from {REPO_ID}...")
26
+ meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
27
+ weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt")
28
+ tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
29
+ print("βœ… Downloaded!")
30
+
31
+ # ╔══════════════════════════════════════════════════════════════════════════╗
32
+ # β•‘ Model Architecture β•‘
33
+ # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
34
+
35
+ def norm(x):
36
+ return F.rms_norm(x, (x.size(-1),))
37
+
38
+ def has_ve(li, nl):
39
+ return li % 2 == (nl - 1) % 2
40
+
41
+ def apply_rotary_emb(x, cos, sin):
42
+ d = x.shape[3] // 2
43
+ x1, x2 = x[..., :d], x[..., d:]
44
+ return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3)
45
+
46
+ class CausalSelfAttention(nn.Module):
47
+ def __init__(self, config, layer_idx):
48
+ super().__init__()
49
+ self.n_head, self.n_kv_head, self.n_embd = config.n_head, config.n_kv_head, config.n_embd
50
+ self.head_dim = self.n_embd // self.n_head
51
+ self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
52
+ self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
53
+ self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
54
+ self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
55
+ self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
56
+
57
+ def forward(self, x, ve, cos_sin, window_size):
58
+ B, T, C = x.size()
59
+ q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
60
+ k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
61
+ v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
62
+ if ve is not None:
63
+ ve = ve.view(B, T, self.n_kv_head, self.head_dim)
64
+ v = v + (2 * torch.sigmoid(self.ve_gate(x[..., :32]))).unsqueeze(-1) * ve
65
+ cos, sin = cos_sin
66
+ q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
67
+ q, k = norm(q), norm(k)
68
+ nr = self.n_head // self.n_kv_head
69
+ if nr > 1:
70
+ k = k.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
71
+ v = v.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
72
+ q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
73
+ wl = window_size[0]
74
+ if 0 < wl < T:
75
+ ri, ci = torch.arange(T, device=x.device).unsqueeze(1), torch.arange(T, device=x.device).unsqueeze(0)
76
+ y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
77
+ else:
78
+ y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
79
+ return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
80
+
81
+ class MLP(nn.Module):
82
+ def __init__(self, config):
83
+ super().__init__()
84
+ self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
85
+ self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
86
+ def forward(self, x):
87
+ return self.c_proj(F.relu(self.c_fc(x)).square())
88
+
89
+ class Block(nn.Module):
90
+ def __init__(self, config, li):
91
+ super().__init__()
92
+ self.attn = CausalSelfAttention(config, li)
93
+ self.mlp = MLP(config)
94
+ def forward(self, x, ve, cs, ws):
95
+ x = x + self.attn(norm(x), ve, cs, ws)
96
+ x = x + self.mlp(norm(x))
97
+ return x
98
+
99
+ @dataclass
100
+ class GPTConfig:
101
+ sequence_len: int = 2048; vocab_size: int = 65536; n_layer: int = 26
102
+ n_head: int = 13; n_kv_head: int = 13; n_embd: int = 1664; window_pattern: str = "L"
103
+
104
+ class GPT(nn.Module):
105
+ def __init__(self, config):
106
+ super().__init__()
107
+ self.config = config
108
+ pat = config.window_pattern.upper()
109
+ lw, sw = config.sequence_len, config.sequence_len // 2
110
+ self.window_sizes = [((sw, 0) if c == 'S' else (lw, 0)) for i, c in enumerate((pat * 4)[:config.n_layer])]
111
+ self.window_sizes[-1] = (lw, 0)
112
+ pad = ((config.vocab_size + 63) // 64) * 64
113
+ self.transformer = nn.ModuleDict({"wte": nn.Embedding(pad, config.n_embd), "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)])})
114
+ self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
115
+ self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))
116
+ self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))
117
+ hd = config.n_embd // config.n_head
118
+ self.value_embeds = nn.ModuleDict({str(i): nn.Embedding(pad, config.n_kv_head * hd) for i in range(config.n_layer) if has_ve(i, config.n_layer)})
119
+ self.rotary_seq_len = config.sequence_len * 10
120
+ inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd))
121
+ freqs = torch.outer(torch.arange(self.rotary_seq_len, dtype=torch.float32), inv)
122
+ self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
123
+ self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
124
+
125
+ def forward(self, idx):
126
+ B, T = idx.size()
127
+ cs = self.cos[:, :T], self.sin[:, :T]
128
+ x = norm(self.transformer.wte(idx)); x0 = x
129
+ for i, b in enumerate(self.transformer.h):
130
+ x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
131
+ ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
132
+ x = b(x, ve, cs, self.window_sizes[i])
133
+ logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
134
+ return 15 * torch.tanh(logits / 15)
135
+
136
+ @torch.inference_mode()
137
+ def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
138
+ dev = next(self.parameters()).device
139
+ ids = torch.tensor([tokens], dtype=torch.long, device=dev)
140
+ for _ in range(max_tokens):
141
+ logits = self.forward(ids[:, -self.config.sequence_len:])[:, -1, :]
142
+ if top_k > 0:
143
+ v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
144
+ logits[logits < v[:, [-1]]] = -float('Inf')
145
+ if temperature > 0:
146
+ nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
147
+ else:
148
+ nid = torch.argmax(logits, dim=-1, keepdim=True)
149
+ ids = torch.cat((ids, nid), dim=1)
150
+ yield nid.item()
151
+
152
+ # ╔══════════════════════════════════════════════════════════════════════════╗
153
+ # β•‘ 8-bit Dequantization Utilities β•‘
154
+ # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
155
+
156
+ def dequantize_state_dict(state_dict):
157
+ """
158
+ Dequantize an 8-bit quantized state dict back to float32.
159
+ Expects int8 weight tensors with corresponding '*_scale' entries.
160
+ Format: weight (int8) + weight_scale (float scalar) β†’ weight = int8 * scale
161
+ """
162
+ dequantized = {}
163
+ scale_keys = {k for k in state_dict if k.endswith('_scale')}
164
+
165
+ for key, tensor in state_dict.items():
166
+ if key in scale_keys:
167
+ continue # skip scale entries, they're consumed with their weight
168
+ if tensor.dtype == torch.int8:
169
+ scale_key = key + '_scale'
170
+ if scale_key in state_dict:
171
+ dequantized[key] = tensor.float() * state_dict[scale_key].float()
172
+ else:
173
+ # Fallback: if no scale found, just cast (shouldn't happen)
174
+ print(f"⚠️ No scale found for {key}, casting directly")
175
+ dequantized[key] = tensor.float()
176
+ else:
177
+ dequantized[key] = tensor
178
+ return dequantized
179
+
180
+ # ╔══════════════════════════════════════════════════════════════════════════╗
181
+ # β•‘ Load Model β•‘
182
+ # β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
183
+
184
+ meta = json.load(open(meta_path))
185
+ mc = meta["model_config"]
186
+ config = GPTConfig(**{k: mc[k] for k in ['sequence_len','vocab_size','n_layer','n_head','n_kv_head','n_embd']})
187
+ config.window_pattern = mc.get("window_pattern", "L")
188
+
189
+ device = "cuda" if torch.cuda.is_available() else "cpu"
190
+ dtype = torch.float16 if device == "cuda" else torch.float32
191
+
192
+ model = GPT(config)
193
+
194
+ # Load 8-bit quantized weights and dequantize
195
+ print("Loading 8-bit quantized weights...")
196
+ raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
197
+
198
+ # Check if any weights are int8 (quantized)
199
+ has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
200
+ if has_int8:
201
+ print("Detected 8-bit quantized weights, dequantizing...")
202
+ state_dict = dequantize_state_dict(raw_state_dict)
203
+ else:
204
+ print("Weights are not int8, loading directly...")
205
+ state_dict = raw_state_dict
206
+
207
+ model.load_state_dict(state_dict)
208
+ model = model.to(device=device, dtype=dtype).eval()
209
+ del raw_state_dict, state_dict # free memory
210
+
211
+ if device == "cuda":
212
+ hd = config.n_embd // config.n_head
213
+ inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32, device=device) / hd))
214
+ freqs = torch.outer(torch.arange(model.rotary_seq_len, dtype=torch.float32, device=device), inv)
215
+ model.cos = freqs.cos()[None, :, None, :]
216
+ model.sin = freqs.sin()[None, :, None, :]
217
+
218
+ with open(tokenizer_path, "rb") as f:
219
+ enc = pickle.load(f)
220
+
221
+ BOS = enc.encode_single_token("<|bos|>")
222
+ USER_START = enc.encode_single_token("<|user_start|>")
223
+ USER_END = enc.encode_single_token("<|user_end|>")
224
+ ASST_START = enc.encode_single_token("<|assistant_start|>")
225
+ ASST_END = enc.encode_single_token("<|assistant_end|>")
226
+
227
+ param_count = sum(p.numel() for p in model.parameters()) / 1e9
228
+ print(f"βœ… Loaded on {device} ({param_count:.1f}B params, dtype={dtype})")
229
+
230
+ def respond(message, chat_history):
231
+ if not message.strip():
232
+ yield ""
233
+ return
234
+
235
+ # Build prompt from messages-format history
236
+ ids = [BOS]
237
+ for entry in chat_history:
238
+ role, content = entry["role"], entry["content"]
239
+ if role == "user":
240
+ ids += [USER_START] + enc.encode_ordinary(content) + [USER_END]
241
+ elif role == "assistant":
242
+ ids += [ASST_START] + enc.encode_ordinary(content) + [ASST_END]
243
+ ids += [USER_START] + enc.encode_ordinary(message) + [USER_END]
244
+ ids.append(ASST_START)
245
+
246
+ # Generate full response
247
+ generated = []
248
+ for token in model.generate(ids, max_tokens=512, temperature=0.7, top_k=50):
249
+ if token == ASST_END:
250
+ break
251
+ generated.append(token)
252
+
253
+ response = enc.decode(generated)
254
+ yield response
255
+
256
+ demo = gr.ChatInterface(
257
+ respond,
258
+ type="messages",
259
+ title="🌳 Treevik-2 2.5B Instruct",
260
+ description="**By PlusB Intelligence Lab** β€’ Created by Sayan Chowdhury\n\n*2.5B parameter model (8-bit quantized) β€” may produce incorrect information.*",
261
+ theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue"),
262
+ chatbot=gr.Chatbot(height=500),
263
+ textbox=gr.Textbox(placeholder="Type a message and press Enter...", container=False, scale=7),
264
+ )
265
+
266
+ demo.launch(server_name="0.0.0.0", server_port=7860)
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ torch
2
+ tiktoken
3
+ gradio
4
+ huggingface_hub