PlusBIntelligenceLab commited on
Commit
2a6e53a
Β·
verified Β·
1 Parent(s): c0fbe17

Updated UI

Browse files
Files changed (1) hide show
  1. app.py +161 -16
app.py CHANGED
@@ -54,7 +54,7 @@ class CausalSelfAttention(nn.Module):
54
  self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
55
  self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
56
 
57
- def forward(self, x, ve, cos_sin, window_size):
58
  B, T, C = x.size()
59
  q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
60
  k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
@@ -65,17 +65,36 @@ class CausalSelfAttention(nn.Module):
65
  cos, sin = cos_sin
66
  q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
67
  q, k = norm(q), norm(k)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  nr = self.n_head // self.n_kv_head
69
  if nr > 1:
70
- k = k.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
71
- v = v.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
72
  q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
73
  wl = window_size[0]
 
74
  if 0 < wl < T:
75
- ri, ci = torch.arange(T, device=x.device).unsqueeze(1), torch.arange(T, device=x.device).unsqueeze(0)
 
 
76
  y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
77
  else:
78
- y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
79
  return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
80
 
81
  class MLP(nn.Module):
@@ -91,8 +110,8 @@ class Block(nn.Module):
91
  super().__init__()
92
  self.attn = CausalSelfAttention(config, li)
93
  self.mlp = MLP(config)
94
- def forward(self, x, ve, cs, ws):
95
- x = x + self.attn(norm(x), ve, cs, ws)
96
  x = x + self.mlp(norm(x))
97
  return x
98
 
@@ -122,14 +141,19 @@ class GPT(nn.Module):
122
  self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
123
  self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
124
 
125
- def forward(self, idx):
126
  B, T = idx.size()
127
- cs = self.cos[:, :T], self.sin[:, :T]
 
128
  x = norm(self.transformer.wte(idx)); x0 = x
129
  for i, b in enumerate(self.transformer.h):
130
  x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
131
  ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
132
- x = b(x, ve, cs, self.window_sizes[i])
 
 
 
 
133
  logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
134
  return 15 * torch.tanh(logits / 15)
135
 
@@ -137,8 +161,26 @@ class GPT(nn.Module):
137
  def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
138
  dev = next(self.parameters()).device
139
  ids = torch.tensor([tokens], dtype=torch.long, device=dev)
 
 
 
 
 
140
  for _ in range(max_tokens):
141
- logits = self.forward(ids[:, -self.config.sequence_len:])[:, -1, :]
 
 
 
 
 
 
 
 
 
 
 
 
 
142
  if top_k > 0:
143
  v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
144
  logits[logits < v[:, [-1]]] = -float('Inf')
@@ -146,7 +188,14 @@ class GPT(nn.Module):
146
  nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
147
  else:
148
  nid = torch.argmax(logits, dim=-1, keepdim=True)
 
149
  ids = torch.cat((ids, nid), dim=1)
 
 
 
 
 
 
150
  yield nid.item()
151
 
152
  # ╔══════════════════════════════════════════════════════════════════════════╗
@@ -253,14 +302,110 @@ def respond(message, chat_history):
253
  response = enc.decode(generated)
254
  yield response
255
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
256
  demo = gr.ChatInterface(
257
  respond,
258
  type="messages",
259
- title="🌳 Treevik-2 2.5B Instruct",
260
- description="**By PlusB Intelligence Lab** β€’ Created by Sayan Chowdhury\n\n*2.5B parameter model (8-bit quantized) β€” may produce incorrect information.*",
261
- theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue"),
262
- chatbot=gr.Chatbot(height=500),
263
- textbox=gr.Textbox(placeholder="Type a message and press Enter...", container=False, scale=7),
 
 
 
 
 
 
 
 
 
 
264
  )
265
 
266
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
54
  self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
55
  self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
56
 
57
+ def forward(self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0):
58
  B, T, C = x.size()
59
  q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
60
  k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
 
65
  cos, sin = cos_sin
66
  q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
67
  q, k = norm(q), norm(k)
68
+
69
+ # KV Cache logic
70
+ if kv_cache is not None:
71
+ if position_id == 0:
72
+ # First token: initialize cache
73
+ kv_cache["k"] = k
74
+ kv_cache["v"] = v
75
+ else:
76
+ # Subsequent tokens: append to cache
77
+ kv_cache["k"] = torch.cat([kv_cache["k"], k], dim=1)
78
+ kv_cache["v"] = torch.cat([kv_cache["v"], v], dim=1)
79
+
80
+ # Use the full cached history for attention
81
+ k = kv_cache["k"]
82
+ v = kv_cache["v"]
83
+
84
  nr = self.n_head // self.n_kv_head
85
  if nr > 1:
86
+ k = k.unsqueeze(3).expand(B, k.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, k.size(1), self.n_head, self.head_dim)
87
+ v = v.unsqueeze(3).expand(B, v.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, v.size(1), self.n_head, self.head_dim)
88
  q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
89
  wl = window_size[0]
90
+
91
  if 0 < wl < T:
92
+ # We only need this branch if T is large, and we're not just predicting one token.
93
+ # Local attention is mostly useful during pre-fill.
94
+ ri, ci = torch.arange(T, device=x.device).unsqueeze(1) + position_id, torch.arange(k.size(2), device=x.device).unsqueeze(0)
95
  y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
96
  else:
97
+ y = F.scaled_dot_product_attention(q, k, v, is_causal=(T > 1))
98
  return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
99
 
100
  class MLP(nn.Module):
 
110
  super().__init__()
111
  self.attn = CausalSelfAttention(config, li)
112
  self.mlp = MLP(config)
113
+ def forward(self, x, ve, cs, ws, kv_cache=None, position_id=0):
114
+ x = x + self.attn(norm(x), ve, cs, ws, kv_cache, position_id)
115
  x = x + self.mlp(norm(x))
116
  return x
117
 
 
141
  self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
142
  self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
143
 
144
+ def forward(self, idx, kv_cache=None, position_id=0):
145
  B, T = idx.size()
146
+ # Adjusted cos/sin slicing based on position
147
+ cs = self.cos[:, position_id:position_id+T], self.sin[:, position_id:position_id+T]
148
  x = norm(self.transformer.wte(idx)); x0 = x
149
  for i, b in enumerate(self.transformer.h):
150
  x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
151
  ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
152
+
153
+ # Extract layer-specific cache or create one if it doesn't exist
154
+ layer_cache = kv_cache[i] if kv_cache is not None else None
155
+
156
+ x = b(x, ve, cs, self.window_sizes[i], layer_cache, position_id)
157
  logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
158
  return 15 * torch.tanh(logits / 15)
159
 
 
161
  def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
162
  dev = next(self.parameters()).device
163
  ids = torch.tensor([tokens], dtype=torch.long, device=dev)
164
+
165
+ # Initialize KV Cache list: one dict per layer
166
+ kv_cache = [{} for _ in range(self.config.n_layer)]
167
+ position_id = 0
168
+
169
  for _ in range(max_tokens):
170
+ # If position_id == 0, pass all tokens (pre-fill)
171
+ # Otherwise, pass ONLY the last token and rely on the cache
172
+ curr_ids = ids if position_id == 0 else ids[:, -1:]
173
+
174
+ # Ensure we don't go past maximum sequence length
175
+ if curr_ids.size(1) + position_id > self.config.sequence_len:
176
+ # Cache eviction logic is missing here for simplicity.
177
+ # In a robust implementation, you'd slice the cache to sequence_len - 1
178
+ curr_ids = curr_ids[:, -self.config.sequence_len:]
179
+ kv_cache = None # Fallback to no cache if we exceed window fully
180
+ position_id = 0
181
+
182
+ logits = self.forward(curr_ids, kv_cache=kv_cache, position_id=position_id)[:, -1, :]
183
+
184
  if top_k > 0:
185
  v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
186
  logits[logits < v[:, [-1]]] = -float('Inf')
 
188
  nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
189
  else:
190
  nid = torch.argmax(logits, dim=-1, keepdim=True)
191
+
192
  ids = torch.cat((ids, nid), dim=1)
193
+
194
+ # Update position_id for the next token
195
+ position_id += curr_ids.size(1)
196
+ if kv_cache is None:
197
+ position_id = 0
198
+
199
  yield nid.item()
200
 
201
  # ╔══════════════════════════════════════════════════════════════════════════╗
 
302
  response = enc.decode(generated)
303
  yield response
304
 
305
+ css = """
306
+ body {
307
+ background-color: #0b0f19;
308
+ color: #e2e8f0;
309
+ }
310
+ .gradio-container {
311
+ max-width: 900px !important;
312
+ border: 1px solid #1e293b;
313
+ border-radius: 12px;
314
+ box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.5), 0 2px 4px -1px rgba(0, 0, 0, 0.3);
315
+ overflow: hidden;
316
+ }
317
+ header, footer {
318
+ display: none !important;
319
+ }
320
+ .message-wrap {
321
+ border-radius: 8px;
322
+ margin-bottom: 8px;
323
+ }
324
+ .message-row.user-row {
325
+ justify-content: flex-end;
326
+ }
327
+ .message-row.bot-row {
328
+ justify-content: flex-start;
329
+ }
330
+ .message {
331
+ padding: 12px 16px;
332
+ border-radius: 12px;
333
+ font-size: 15px;
334
+ line-height: 1.5;
335
+ box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.05);
336
+ }
337
+ .message.user {
338
+ background-color: #1e293b;
339
+ color: #f8fafc;
340
+ border-bottom-right-radius: 4px;
341
+ }
342
+ .message.bot {
343
+ background-color: #0f172a;
344
+ color: #cbd5e1;
345
+ border-bottom-left-radius: 4px;
346
+ }
347
+ textarea {
348
+ background-color: #0f172a !important;
349
+ border: 1px solid #334155 !important;
350
+ color: #f8fafc !important;
351
+ border-radius: 12px !important;
352
+ }
353
+ textarea:focus {
354
+ border-color: #3b82f6 !important;
355
+ box-shadow: 0 0 0 1px #3b82f6 !important;
356
+ }
357
+ button.primary {
358
+ background: linear-gradient(135deg, #3b82f6, #2563eb);
359
+ color: white !important;
360
+ border-radius: 8px !important;
361
+ border: none !important;
362
+ }
363
+ .panel {
364
+ background-color: #0b0f19;
365
+ border: none !important;
366
+ }
367
+ """
368
+
369
+ theme = gr.themes.Base(
370
+ primary_hue="blue",
371
+ secondary_hue="slate",
372
+ neutral_hue="slate",
373
+ spacing_size="sm",
374
+ radius_size="lg",
375
+ font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
376
+ ).set(
377
+ body_background_fill="#0b0f19",
378
+ body_text_color="#f8fafc",
379
+ background_fill_primary="#0b0f19",
380
+ background_fill_secondary="#0f172a",
381
+ border_color_primary="#1e293b",
382
+ color_accent="#3b82f6",
383
+ button_primary_background_fill="*color_accent",
384
+ button_primary_background_fill_hover="#2563eb",
385
+ block_background_fill="#0f172a",
386
+ block_border_width="1px",
387
+ block_border_color="#1e293b",
388
+ input_background_fill="#0f172a",
389
+ )
390
+
391
  demo = gr.ChatInterface(
392
  respond,
393
  type="messages",
394
+ title="Treevik-2 (2.5B)",
395
+ description="""
396
+ <div style="text-align: center; margin-top: -10px; margin-bottom: 20px;">
397
+ <p style="color: #64748b; font-size: 14px; margin: 0; font-weight: 500; letter-spacing: 0.5px; text-transform: uppercase;">PlusB Intelligence Lab</p>
398
+ <p style="color: #475569; font-size: 12px; margin-top: 4px;">Instruction-tuned 2.5B Parameter Model β€’ 8-bit Quantized</p>
399
+ </div>
400
+ """,
401
+ theme=theme,
402
+ css=css,
403
+ chatbot=gr.Chatbot(height=550, show_label=False, show_copy_button=True, avatar_images=(None, "https://ui-avatars.com/api/?name=PB&background=2563eb&color=fff&size=256&font-size=0.4&bold=true")),
404
+ textbox=gr.Textbox(placeholder="Ask Treevik anything...", container=False, scale=7, min_lines=1, max_lines=5),
405
+ submit_btn=gr.Button("Send", variant="primary", scale=1),
406
+ retry_btn=None,
407
+ undo_btn=None,
408
+ clear_btn=gr.Button("Clear Chat", variant="secondary", size="sm"),
409
  )
410
 
411
  demo.launch(server_name="0.0.0.0", server_port=7860)