PlusBIntelligenceLab commited on
Commit
97a7be4
·
verified ·
1 Parent(s): e1b270d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +259 -299
app.py CHANGED
@@ -1,36 +1,55 @@
1
  """
2
- Treevik-2 2.5B Instruct (8-bit Quantized) — HuggingFace Space
3
- Downloads model from HF Hub: PlusBIntelligenceLab/Treevik2-2.5B-Instruct
 
4
  """
5
 
6
- import os
7
  import json
8
  import pickle
9
  from dataclasses import dataclass
10
 
 
 
11
  import torch
12
  import torch.nn as nn
13
  import torch.nn.functional as F
14
- import gradio as gr
15
  from huggingface_hub import hf_hub_download
16
 
 
 
 
17
 
 
18
 
19
- # ╔══════════════════════════════════════════════════════════════════════════╗
20
- # ║ Download Model from HuggingFace Hub ║
21
- # ╚══════════════════════════════════════════════════════════════════════════╝
22
 
23
- REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
 
 
 
 
 
 
 
 
 
 
 
 
 
24
 
25
  print(f"Downloading model from {REPO_ID}...")
 
26
  meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
27
  weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt")
28
  tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
 
29
  print("✅ Downloaded!")
30
 
31
- # ╔══════════════════════════════════════════════════════════════════════════╗
32
- # ║ Model Architecture ║
33
- # ╚══════════════════════════════════════════════════════════════════════════╝
34
 
35
  def norm(x):
36
  return F.rms_norm(x, (x.size(-1),))
@@ -46,70 +65,103 @@ def apply_rotary_emb(x, cos, sin):
46
  class CausalSelfAttention(nn.Module):
47
  def __init__(self, config, layer_idx):
48
  super().__init__()
49
- self.n_head, self.n_kv_head, self.n_embd = config.n_head, config.n_kv_head, config.n_embd
 
 
 
50
  self.head_dim = self.n_embd // self.n_head
 
51
  self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
52
  self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
53
  self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
 
54
  self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
55
- self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
 
 
 
 
 
56
 
57
  def forward(self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0):
58
  B, T, C = x.size()
 
59
  q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
60
  k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
61
  v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
 
62
  if ve is not None:
63
  ve = ve.view(B, T, self.n_kv_head, self.head_dim)
64
- v = v + (2 * torch.sigmoid(self.ve_gate(x[..., :32]))).unsqueeze(-1) * ve
 
 
 
65
  cos, sin = cos_sin
66
- q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
67
- q, k = norm(q), norm(k)
68
-
69
- # KV Cache logic
 
 
 
70
  if kv_cache is not None:
71
  if position_id == 0:
72
- # First token: initialize cache
73
  kv_cache["k"] = k
74
  kv_cache["v"] = v
75
  else:
76
- # Subsequent tokens: append to cache
77
  kv_cache["k"] = torch.cat([kv_cache["k"], k], dim=1)
78
  kv_cache["v"] = torch.cat([kv_cache["v"], v], dim=1)
79
-
80
- # Use the full cached history for attention
81
  k = kv_cache["k"]
82
  v = kv_cache["v"]
83
 
84
  nr = self.n_head // self.n_kv_head
 
85
  if nr > 1:
86
- k = k.unsqueeze(3).expand(B, k.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, k.size(1), self.n_head, self.head_dim)
87
- v = v.unsqueeze(3).expand(B, v.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, v.size(1), self.n_head, self.head_dim)
88
- q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
89
- wl = window_size[0]
90
-
91
- if 0 < wl < T:
92
- # We only need this branch if T is large, and we're not just predicting one token.
93
- # Local attention is mostly useful during pre-fill.
94
- ri, ci = torch.arange(T, device=x.device).unsqueeze(1) + position_id, torch.arange(k.size(2), device=x.device).unsqueeze(0)
95
- y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
96
- else:
97
- y = F.scaled_dot_product_attention(q, k, v, is_causal=(T > 1))
98
- return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
 
 
 
 
 
 
 
 
 
 
 
 
 
99
 
100
  class MLP(nn.Module):
101
  def __init__(self, config):
102
  super().__init__()
 
103
  self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
104
  self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
 
105
  def forward(self, x):
106
  return self.c_proj(F.relu(self.c_fc(x)).square())
107
 
108
  class Block(nn.Module):
109
  def __init__(self, config, li):
110
  super().__init__()
 
111
  self.attn = CausalSelfAttention(config, li)
112
  self.mlp = MLP(config)
 
113
  def forward(self, x, ve, cs, ws, kv_cache=None, position_id=0):
114
  x = x + self.attn(norm(x), ve, cs, ws, kv_cache, position_id)
115
  x = x + self.mlp(norm(x))
@@ -117,191 +169,166 @@ class Block(nn.Module):
117
 
118
  @dataclass
119
  class GPTConfig:
120
- sequence_len: int = 2048; vocab_size: int = 65536; n_layer: int = 26
121
- n_head: int = 13; n_kv_head: int = 13; n_embd: int = 1664; window_pattern: str = "L"
 
 
 
 
 
122
 
123
  class GPT(nn.Module):
124
  def __init__(self, config):
125
  super().__init__()
 
126
  self.config = config
127
- pat = config.window_pattern.upper()
128
- lw, sw = config.sequence_len, config.sequence_len // 2
129
- self.window_sizes = [((sw, 0) if c == 'S' else (lw, 0)) for i, c in enumerate((pat * config.n_layer)[:config.n_layer])]
130
- self.window_sizes[-1] = (lw, 0)
131
  pad = ((config.vocab_size + 63) // 64) * 64
132
- self.transformer = nn.ModuleDict({"wte": nn.Embedding(pad, config.n_embd), "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)])})
 
 
 
 
 
 
 
133
  self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
134
- self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))
135
- self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))
136
  hd = config.n_embd // config.n_head
137
- self.value_embeds = nn.ModuleDict({str(i): nn.Embedding(pad, config.n_kv_head * hd) for i in range(config.n_layer) if has_ve(i, config.n_layer)})
138
- self.rotary_seq_len = config.sequence_len * 10
139
- inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd))
140
- freqs = torch.outer(torch.arange(self.rotary_seq_len, dtype=torch.float32), inv)
141
- self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
142
- self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
143
-
144
- def forward(self, idx, kv_cache=None, position_id=0):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  B, T = idx.size()
146
- # Adjusted cos/sin slicing based on position
147
- cs = self.cos[:, position_id:position_id+T], self.sin[:, position_id:position_id+T]
148
- x = norm(self.transformer.wte(idx)); x0 = x
149
- for i, b in enumerate(self.transformer.h):
150
- x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
151
- ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
152
-
153
- # Extract layer-specific cache or create one if it doesn't exist
154
- layer_cache = kv_cache[i] if kv_cache is not None else None
155
-
156
- x = b(x, ve, cs, self.window_sizes[i], layer_cache, position_id)
157
- logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
158
- return 15 * torch.tanh(logits / 15)
159
 
160
  @torch.inference_mode()
161
- def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
162
- dev = next(self.parameters()).device
163
- ids = torch.tensor([tokens], dtype=torch.long, device=dev)
164
-
165
- # Initialize KV Cache list: one dict per layer
166
- kv_cache = [{} for _ in range(self.config.n_layer)]
167
- position_id = 0
168
-
169
  for _ in range(max_tokens):
170
- # If position_id == 0, pass all tokens (pre-fill)
171
- # Otherwise, pass ONLY the last token and rely on the cache
172
- curr_ids = ids if position_id == 0 else ids[:, -1:]
173
-
174
- # Ensure we don't go past maximum sequence length
175
- if curr_ids.size(1) + position_id > self.config.sequence_len:
176
- # Cache eviction logic is missing here for simplicity.
177
- # In a robust implementation, you'd slice the cache to sequence_len - 1
178
- curr_ids = curr_ids[:, -self.config.sequence_len:]
179
- kv_cache = None # Fallback to no cache if we exceed window fully
180
- position_id = 0
181
-
182
- logits = self.forward(curr_ids, kv_cache=kv_cache, position_id=position_id)[:, -1, :]
183
-
184
  if top_k > 0:
185
- v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
186
- logits[logits < v[:, [-1]]] = -float('Inf')
187
- if temperature > 0:
188
- nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
189
- else:
190
- nid = torch.argmax(logits, dim=-1, keepdim=True)
191
-
 
 
 
192
  ids = torch.cat((ids, nid), dim=1)
193
-
194
- # Update position_id for the next token
195
- position_id += curr_ids.size(1)
196
- if kv_cache is None:
197
- position_id = 0
198
-
199
- yield nid.item()
200
 
201
- # ╔══════════════════════════════════════════════════════════════════════════╗
202
- # ║ 8-bit Dequantization Utilities ║
203
- # ╚══════════════════════════════════════════════════════════════════════════╝
204
-
205
- def dequantize_state_dict(state_dict):
206
- """
207
- Dequantize an 8-bit quantized state dict back to float32.
208
- Expects int8 weight tensors with corresponding '*_scale' entries.
209
- Format: weight (int8) + weight_scale (float scalar) → weight = int8 * scale
210
- """
211
- dequantized = {}
212
- scale_keys = {k for k in state_dict if k.endswith('_scale')}
213
-
214
- for key, tensor in state_dict.items():
215
- if key in scale_keys:
216
- continue # skip scale entries, they're consumed with their weight
217
- if tensor.dtype == torch.int8:
218
- scale_key = key + '_scale'
219
- if scale_key in state_dict:
220
- dequantized[key] = tensor.float() * state_dict[scale_key].float()
221
- else:
222
- # Fallback: if no scale found, just cast (shouldn't happen)
223
- print(f"⚠️ No scale found for {key}, casting directly")
224
- dequantized[key] = tensor.float()
225
- else:
226
- dequantized[key] = tensor
227
- return dequantized
228
 
229
- # ╔══════════════════════════════════════════════════════════════════════════╗
230
- # ║ 8-bit Dequantization Utilities & Model Loading ║
231
- # ╚══════════════════════════════════════════════════════════════════════════╝
232
 
233
  meta = json.load(open(meta_path))
 
234
  mc = meta["model_config"]
235
- config = GPTConfig(**{k: mc[k] for k in ['sequence_len','vocab_size','n_layer','n_head','n_kv_head','n_embd']})
236
- config.window_pattern = mc.get("window_pattern", "L")
237
 
238
- device = "cuda" if torch.cuda.is_available() else "cpu"
239
- dtype = torch.float16 if device == "cuda" else torch.float32
 
 
 
 
 
 
 
 
 
240
 
241
  model = GPT(config)
242
- model = model.to(device=device, dtype=dtype)
243
 
244
- print("Loading 8-bit quantized weights...")
245
- raw_state_dict = torch.load(weights_path, map_location="cpu", weights_only=True)
246
 
247
- # Delete large downloaded file references to free memory early if possible
248
- import gc
249
- gc.collect()
 
 
250
 
251
- has_int8 = any(v.dtype == torch.int8 for v in raw_state_dict.values())
252
-
253
- if has_int8:
254
- print("Detected 8-bit quantized weights. Dequantizing directly into model to save RAM...")
255
-
256
- # We will iterate through the model's named parameters and pull from raw_state_dict
257
- # This prevents creating a second massive dict in memory.
258
- model_state = model.state_dict()
259
-
260
- for key in list(model_state.keys()):
261
- if key in raw_state_dict:
262
- tensor = raw_state_dict[key]
263
-
264
- if tensor.dtype == torch.int8:
265
- scale_key = key + '_scale'
266
- if scale_key in raw_state_dict:
267
- # Dequantize directly into the target dtype
268
- dequantized = tensor.to(dtype) * raw_state_dict[scale_key].to(dtype)
269
- model_state[key].copy_(dequantized)
270
-
271
- # Delete the scale from raw dict to free memory as we go
272
- del raw_state_dict[scale_key]
273
- else:
274
- print(f"⚠️ No scale found for {key}, casting directly")
275
- model_state[key].copy_(tensor.to(dtype))
276
- else:
277
- model_state[key].copy_(tensor.to(dtype))
278
-
279
- # Free the tensor from raw_state_dict as soon as we're done with it
280
- del raw_state_dict[key]
281
-
282
- # Ensure any remaining keys in raw_state_dict (like buffers not in model_state) are also copied
283
- # though this shouldn't usually happen for standard SFT weights
284
- for key in list(raw_state_dict.keys()):
285
- if key not in model_state and not key.endswith('_scale'):
286
- pass # Ignore extra keys
287
-
288
- print("Dequantization complete.")
289
 
290
- else:
291
- print("Weights are not int8, loading directly...")
292
- model.load_state_dict(raw_state_dict)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
293
 
294
  del raw_state_dict
295
  gc.collect()
296
 
297
- model.eval()
 
298
 
299
- if device == "cuda":
300
- hd = config.n_embd // config.n_head
301
- inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32, device=device) / hd))
302
- freqs = torch.outer(torch.arange(model.rotary_seq_len, dtype=torch.float32, device=device), inv)
303
- model.cos = freqs.cos()[None, :, None, :]
304
- model.sin = freqs.sin()[None, :, None, :]
305
 
306
  with open(tokenizer_path, "rb") as f:
307
  enc = pickle.load(f)
@@ -313,129 +340,62 @@ ASST_START = enc.encode_single_token("<|assistant_start|>")
313
  ASST_END = enc.encode_single_token("<|assistant_end|>")
314
 
315
  param_count = sum(p.numel() for p in model.parameters()) / 1e9
316
- print(f"✅ Loaded on {device} ({param_count:.1f}B params, dtype={dtype})")
317
 
318
- def respond(message, chat_history):
 
 
 
 
 
 
 
 
 
 
319
  if not message.strip():
320
  yield ""
321
  return
322
 
323
- # Build prompt from current message only (no history)
324
  ids = [BOS]
325
- ids += [USER_START] + enc.encode_ordinary(message) + [USER_END]
326
- ids.append(ASST_START)
 
 
 
 
 
 
 
 
 
 
 
327
 
328
- # Generate full response
329
- generated = []
330
- for token in model.generate(ids, max_tokens=512, temperature=0.7, top_k=50):
331
  if token == ASST_END:
332
  break
333
- generated.append(token)
334
-
335
- response = enc.decode(generated)
336
- yield response
337
-
338
- css = """
339
- body {
340
- background-color: #0b0f19;
341
- color: #e2e8f0;
342
- }
343
- .gradio-container {
344
- max-width: 900px !important;
345
- border: 1px solid #1e293b;
346
- border-radius: 12px;
347
- box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.5), 0 2px 4px -1px rgba(0, 0, 0, 0.3);
348
- overflow: hidden;
349
- }
350
- header, footer {
351
- display: none !important;
352
- }
353
- .message-wrap {
354
- border-radius: 8px;
355
- margin-bottom: 8px;
356
- }
357
- .message-row.user-row {
358
- justify-content: flex-end;
359
- }
360
- .message-row.bot-row {
361
- justify-content: flex-start;
362
- }
363
- .message {
364
- padding: 12px 16px;
365
- border-radius: 12px;
366
- font-size: 15px;
367
- line-height: 1.5;
368
- box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.05);
369
- }
370
- .message.user {
371
- background-color: #1e293b;
372
- color: #f8fafc;
373
- border-bottom-right-radius: 4px;
374
- }
375
- .message.bot {
376
- background-color: #0f172a;
377
- color: #cbd5e1;
378
- border-bottom-left-radius: 4px;
379
- }
380
- textarea {
381
- background-color: #0f172a !important;
382
- border: 1px solid #334155 !important;
383
- color: #f8fafc !important;
384
- border-radius: 12px !important;
385
- }
386
- textarea:focus {
387
- border-color: #3b82f6 !important;
388
- box-shadow: 0 0 0 1px #3b82f6 !important;
389
- }
390
- button.primary {
391
- background: linear-gradient(135deg, #3b82f6, #2563eb);
392
- color: white !important;
393
- border-radius: 8px !important;
394
- border: none !important;
395
- }
396
- .panel {
397
- background-color: #0b0f19;
398
- border: none !important;
399
- }
400
- """
401
 
402
- theme = gr.themes.Base(
403
- primary_hue="blue",
404
- secondary_hue="slate",
405
- neutral_hue="slate",
406
- spacing_size="sm",
407
- radius_size="lg",
408
- font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
409
- ).set(
410
- body_background_fill="#0b0f19",
411
- body_text_color="#f8fafc",
412
- background_fill_primary="#0b0f19",
413
- background_fill_secondary="#0f172a",
414
- border_color_primary="#1e293b",
415
- color_accent="#3b82f6",
416
- button_primary_background_fill="*color_accent",
417
- button_primary_background_fill_hover="#2563eb",
418
- block_background_fill="#0f172a",
419
- block_border_width="1px",
420
- block_border_color="#1e293b",
421
- input_background_fill="#0f172a",
422
- )
423
 
424
  demo = gr.ChatInterface(
425
- respond,
426
  type="messages",
427
  title="Treevik-2 (2.5B)",
428
- description="""
429
- <div style="text-align: center; margin-top: -10px; margin-bottom: 20px;">
430
- <p style="color: #64748b; font-size: 14px; margin: 0; font-weight: 500; letter-spacing: 0.5px; text-transform: uppercase;">PlusB Intelligence Lab</p>
431
- <p style="color: #475569; font-size: 12px; margin-top: 4px;">Instruction-tuned 2.5B Parameter Model • 8-bit Quantized</p>
432
- </div>
433
- """,
434
- theme=theme,
435
- css=css,
436
- chatbot=gr.Chatbot(height=550, show_label=False, show_copy_button=True, avatar_images=(None, "https://ui-avatars.com/api/?name=PB&background=2563eb&color=fff&size=256&font-size=0.4&bold=true")),
437
- textbox=gr.Textbox(placeholder="Ask Treevik anything...", container=False, scale=7, max_lines=5),
438
- submit_btn=gr.Button("Send", variant="primary", scale=1),
439
  )
440
 
441
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
 
 
 
1
  """
2
+ Treevik-2 2.5B Instruct (HF Spaces Ready)
3
+ Auto-detects GPU/CPU automatically.
4
+ Uses GPU if available, otherwise CPU fallback.
5
  """
6
 
7
+ import gc
8
  import json
9
  import pickle
10
  from dataclasses import dataclass
11
 
12
+ import gradio as gr
13
+ import spaces
14
  import torch
15
  import torch.nn as nn
16
  import torch.nn.functional as F
 
17
  from huggingface_hub import hf_hub_download
18
 
19
+ # ============================================================================
20
+ # SETTINGS
21
+ # ============================================================================
22
 
23
+ REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
24
 
25
+ torch.set_grad_enabled(False)
 
 
26
 
27
+ if torch.cuda.is_available():
28
+ DEVICE = "cuda"
29
+ DTYPE = torch.float16
30
+ torch.backends.cuda.matmul.allow_tf32 = True
31
+ else:
32
+ DEVICE = "cpu"
33
+ DTYPE = torch.float16
34
+
35
+ print(f"Using device: {DEVICE}")
36
+ print(f"Using dtype: {DTYPE}")
37
+
38
+ # ============================================================================
39
+ # DOWNLOAD MODEL
40
+ # ============================================================================
41
 
42
  print(f"Downloading model from {REPO_ID}...")
43
+
44
  meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
45
  weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt")
46
  tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
47
+
48
  print("✅ Downloaded!")
49
 
50
+ # ============================================================================
51
+ # MODEL ARCHITECTURE
52
+ # ============================================================================
53
 
54
  def norm(x):
55
  return F.rms_norm(x, (x.size(-1),))
 
65
  class CausalSelfAttention(nn.Module):
66
  def __init__(self, config, layer_idx):
67
  super().__init__()
68
+
69
+ self.n_head = config.n_head
70
+ self.n_kv_head = config.n_kv_head
71
+ self.n_embd = config.n_embd
72
  self.head_dim = self.n_embd // self.n_head
73
+
74
  self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
75
  self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
76
  self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
77
+
78
  self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
79
+
80
+ self.ve_gate = (
81
+ nn.Linear(32, self.n_kv_head, bias=False)
82
+ if has_ve(layer_idx, config.n_layer)
83
+ else None
84
+ )
85
 
86
  def forward(self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0):
87
  B, T, C = x.size()
88
+
89
  q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
90
  k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
91
  v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
92
+
93
  if ve is not None:
94
  ve = ve.view(B, T, self.n_kv_head, self.head_dim)
95
+ v = v + (
96
+ 2 * torch.sigmoid(self.ve_gate(x[..., :32]))
97
+ ).unsqueeze(-1) * ve
98
+
99
  cos, sin = cos_sin
100
+
101
+ q = apply_rotary_emb(q, cos, sin)
102
+ k = apply_rotary_emb(k, cos, sin)
103
+
104
+ q = norm(q)
105
+ k = norm(k)
106
+
107
  if kv_cache is not None:
108
  if position_id == 0:
 
109
  kv_cache["k"] = k
110
  kv_cache["v"] = v
111
  else:
 
112
  kv_cache["k"] = torch.cat([kv_cache["k"], k], dim=1)
113
  kv_cache["v"] = torch.cat([kv_cache["v"], v], dim=1)
114
+
 
115
  k = kv_cache["k"]
116
  v = kv_cache["v"]
117
 
118
  nr = self.n_head // self.n_kv_head
119
+
120
  if nr > 1:
121
+ k = (
122
+ k.unsqueeze(3)
123
+ .expand(B, k.size(1), self.n_kv_head, nr, self.head_dim)
124
+ .reshape(B, k.size(1), self.n_head, self.head_dim)
125
+ )
126
+
127
+ v = (
128
+ v.unsqueeze(3)
129
+ .expand(B, v.size(1), self.n_kv_head, nr, self.head_dim)
130
+ .reshape(B, v.size(1), self.n_head, self.head_dim)
131
+ )
132
+
133
+ q = q.transpose(1, 2)
134
+ k = k.transpose(1, 2)
135
+ v = v.transpose(1, 2)
136
+
137
+ y = F.scaled_dot_product_attention(
138
+ q,
139
+ k,
140
+ v,
141
+ is_causal=(T > 1),
142
+ )
143
+
144
+ return self.c_proj(
145
+ y.transpose(1, 2).contiguous().view(B, T, -1)
146
+ )
147
 
148
  class MLP(nn.Module):
149
  def __init__(self, config):
150
  super().__init__()
151
+
152
  self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
153
  self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
154
+
155
  def forward(self, x):
156
  return self.c_proj(F.relu(self.c_fc(x)).square())
157
 
158
  class Block(nn.Module):
159
  def __init__(self, config, li):
160
  super().__init__()
161
+
162
  self.attn = CausalSelfAttention(config, li)
163
  self.mlp = MLP(config)
164
+
165
  def forward(self, x, ve, cs, ws, kv_cache=None, position_id=0):
166
  x = x + self.attn(norm(x), ve, cs, ws, kv_cache, position_id)
167
  x = x + self.mlp(norm(x))
 
169
 
170
  @dataclass
171
  class GPTConfig:
172
+ sequence_len: int = 2048
173
+ vocab_size: int = 65536
174
+ n_layer: int = 26
175
+ n_head: int = 13
176
+ n_kv_head: int = 13
177
+ n_embd: int = 1664
178
+ window_pattern: str = "L"
179
 
180
  class GPT(nn.Module):
181
  def __init__(self, config):
182
  super().__init__()
183
+
184
  self.config = config
185
+
 
 
 
186
  pad = ((config.vocab_size + 63) // 64) * 64
187
+
188
+ self.transformer = nn.ModuleDict({
189
+ "wte": nn.Embedding(pad, config.n_embd),
190
+ "h": nn.ModuleList([
191
+ Block(config, i) for i in range(config.n_layer)
192
+ ])
193
+ })
194
+
195
  self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
196
+
 
197
  hd = config.n_embd // config.n_head
198
+
199
+ inv = 1.0 / (
200
+ 10000 ** (
201
+ torch.arange(0, hd, 2, dtype=torch.float32) / hd
202
+ )
203
+ )
204
+
205
+ freqs = torch.outer(
206
+ torch.arange(config.sequence_len * 10, dtype=torch.float32),
207
+ inv
208
+ )
209
+
210
+ self.register_buffer(
211
+ "cos",
212
+ freqs.cos()[None, :, None, :],
213
+ persistent=False,
214
+ )
215
+
216
+ self.register_buffer(
217
+ "sin",
218
+ freqs.sin()[None, :, None, :],
219
+ persistent=False,
220
+ )
221
+
222
+ def forward(self, idx):
223
  B, T = idx.size()
224
+
225
+ cs = (
226
+ self.cos[:, :T],
227
+ self.sin[:, :T],
228
+ )
229
+
230
+ x = norm(self.transformer["wte"](idx))
231
+
232
+ for block in self.transformer["h"]:
233
+ x = block(x, None, cs, (0, 0))
234
+
235
+ logits = self.lm_head(norm(x))
236
+ return logits[..., :self.config.vocab_size]
237
 
238
  @torch.inference_mode()
239
+ def generate(self, tokens, max_tokens=256, temperature=0.7, top_k=50):
240
+
241
+ ids = torch.tensor(
242
+ [tokens],
243
+ dtype=torch.long,
244
+ device=DEVICE,
245
+ )
246
+
247
  for _ in range(max_tokens):
248
+
249
+ logits = self.forward(ids)[:, -1, :]
250
+
 
 
 
 
 
 
 
 
 
 
 
251
  if top_k > 0:
252
+ v, _ = torch.topk(
253
+ logits,
254
+ min(top_k, logits.size(-1))
255
+ )
256
+ logits[logits < v[:, [-1]]] = -float("Inf")
257
+
258
+ probs = F.softmax(logits / temperature, dim=-1)
259
+
260
+ nid = torch.multinomial(probs, num_samples=1)
261
+
262
  ids = torch.cat((ids, nid), dim=1)
 
 
 
 
 
 
 
263
 
264
+ yield nid.item()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
265
 
266
+ # ============================================================================
267
+ # LOAD MODEL
268
+ # ============================================================================
269
 
270
  meta = json.load(open(meta_path))
271
+
272
  mc = meta["model_config"]
 
 
273
 
274
+ config = GPTConfig(**{
275
+ k: mc[k]
276
+ for k in [
277
+ "sequence_len",
278
+ "vocab_size",
279
+ "n_layer",
280
+ "n_head",
281
+ "n_kv_head",
282
+ "n_embd",
283
+ ]
284
+ })
285
 
286
  model = GPT(config)
 
287
 
288
+ print("Loading weights...")
 
289
 
290
+ raw_state_dict = torch.load(
291
+ weights_path,
292
+ map_location="cpu",
293
+ weights_only=True,
294
+ )
295
 
296
+ model_state = model.state_dict()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
297
 
298
+ for key in list(model_state.keys()):
299
+
300
+ if key not in raw_state_dict:
301
+ continue
302
+
303
+ tensor = raw_state_dict[key]
304
+
305
+ if tensor.dtype == torch.int8:
306
+
307
+ scale_key = key + "_scale"
308
+
309
+ if scale_key in raw_state_dict:
310
+ tensor = (
311
+ tensor.float()
312
+ * raw_state_dict[scale_key].float()
313
+ )
314
+
315
+ model_state[key].copy_(tensor.to(model_state[key].dtype))
316
+
317
+ print("Moving model to device...")
318
+
319
+ model = model.to(device=DEVICE, dtype=DTYPE)
320
+
321
+ model.eval()
322
 
323
  del raw_state_dict
324
  gc.collect()
325
 
326
+ if DEVICE == "cuda":
327
+ torch.cuda.empty_cache()
328
 
329
+ # ============================================================================
330
+ # TOKENIZER
331
+ # ============================================================================
 
 
 
332
 
333
  with open(tokenizer_path, "rb") as f:
334
  enc = pickle.load(f)
 
340
  ASST_END = enc.encode_single_token("<|assistant_end|>")
341
 
342
  param_count = sum(p.numel() for p in model.parameters()) / 1e9
 
343
 
344
+ print(
345
+ f"✅ Loaded model ({param_count:.1f}B params) on {DEVICE}"
346
+ )
347
+
348
+ # ============================================================================
349
+ # INFERENCE
350
+ # ============================================================================
351
+
352
+ @spaces.GPU
353
+ def respond(message, history):
354
+
355
  if not message.strip():
356
  yield ""
357
  return
358
 
 
359
  ids = [BOS]
360
+ ids += [USER_START]
361
+ ids += enc.encode_ordinary(message)
362
+ ids += [USER_END]
363
+ ids += [ASST_START]
364
+
365
+ partial = ""
366
+
367
+ for token in model.generate(
368
+ ids,
369
+ max_tokens=256,
370
+ temperature=0.7,
371
+ top_k=50,
372
+ ):
373
 
 
 
 
374
  if token == ASST_END:
375
  break
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
376
 
377
+ piece = enc.decode([token])
378
+
379
+ partial += piece
380
+
381
+ yield partial
382
+
383
+ # ============================================================================
384
+ # UI
385
+ # ============================================================================
 
 
 
 
 
 
 
 
 
 
 
 
386
 
387
  demo = gr.ChatInterface(
388
+ fn=respond,
389
  type="messages",
390
  title="Treevik-2 (2.5B)",
391
+ description="PlusB Intelligence Lab",
392
+ chatbot=gr.Chatbot(height=600),
393
+ textbox=gr.Textbox(
394
+ placeholder="Ask Treevik anything..."
395
+ ),
 
 
 
 
 
 
396
  )
397
 
398
+ demo.launch(
399
+ server_name="0.0.0.0",
400
+ server_port=7860,
401
+ )