Updated UI
Browse files
app.py
CHANGED
|
@@ -54,7 +54,7 @@ class CausalSelfAttention(nn.Module):
|
|
| 54 |
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
|
| 55 |
self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
|
| 56 |
|
| 57 |
-
def forward(self, x, ve, cos_sin, window_size):
|
| 58 |
B, T, C = x.size()
|
| 59 |
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
|
| 60 |
k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
|
|
@@ -65,17 +65,36 @@ class CausalSelfAttention(nn.Module):
|
|
| 65 |
cos, sin = cos_sin
|
| 66 |
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
|
| 67 |
q, k = norm(q), norm(k)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
nr = self.n_head // self.n_kv_head
|
| 69 |
if nr > 1:
|
| 70 |
-
k = k.unsqueeze(3).expand(B,
|
| 71 |
-
v = v.unsqueeze(3).expand(B,
|
| 72 |
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 73 |
wl = window_size[0]
|
|
|
|
| 74 |
if 0 < wl < T:
|
| 75 |
-
|
|
|
|
|
|
|
| 76 |
y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
|
| 77 |
else:
|
| 78 |
-
y = F.scaled_dot_product_attention(q, k, v, is_causal=
|
| 79 |
return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
|
| 80 |
|
| 81 |
class MLP(nn.Module):
|
|
@@ -91,8 +110,8 @@ class Block(nn.Module):
|
|
| 91 |
super().__init__()
|
| 92 |
self.attn = CausalSelfAttention(config, li)
|
| 93 |
self.mlp = MLP(config)
|
| 94 |
-
def forward(self, x, ve, cs, ws):
|
| 95 |
-
x = x + self.attn(norm(x), ve, cs, ws)
|
| 96 |
x = x + self.mlp(norm(x))
|
| 97 |
return x
|
| 98 |
|
|
@@ -122,14 +141,19 @@ class GPT(nn.Module):
|
|
| 122 |
self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
|
| 123 |
self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
|
| 124 |
|
| 125 |
-
def forward(self, idx):
|
| 126 |
B, T = idx.size()
|
| 127 |
-
|
|
|
|
| 128 |
x = norm(self.transformer.wte(idx)); x0 = x
|
| 129 |
for i, b in enumerate(self.transformer.h):
|
| 130 |
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 131 |
ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
|
| 134 |
return 15 * torch.tanh(logits / 15)
|
| 135 |
|
|
@@ -137,8 +161,26 @@ class GPT(nn.Module):
|
|
| 137 |
def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
|
| 138 |
dev = next(self.parameters()).device
|
| 139 |
ids = torch.tensor([tokens], dtype=torch.long, device=dev)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
for _ in range(max_tokens):
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
if top_k > 0:
|
| 143 |
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
|
| 144 |
logits[logits < v[:, [-1]]] = -float('Inf')
|
|
@@ -146,7 +188,14 @@ class GPT(nn.Module):
|
|
| 146 |
nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
|
| 147 |
else:
|
| 148 |
nid = torch.argmax(logits, dim=-1, keepdim=True)
|
|
|
|
| 149 |
ids = torch.cat((ids, nid), dim=1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
yield nid.item()
|
| 151 |
|
| 152 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -253,14 +302,110 @@ def respond(message, chat_history):
|
|
| 253 |
response = enc.decode(generated)
|
| 254 |
yield response
|
| 255 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
demo = gr.ChatInterface(
|
| 257 |
respond,
|
| 258 |
type="messages",
|
| 259 |
-
title="
|
| 260 |
-
description="
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
)
|
| 265 |
|
| 266 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
| 54 |
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
|
| 55 |
self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
|
| 56 |
|
| 57 |
+
def forward(self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0):
|
| 58 |
B, T, C = x.size()
|
| 59 |
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
|
| 60 |
k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
|
|
|
|
| 65 |
cos, sin = cos_sin
|
| 66 |
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
|
| 67 |
q, k = norm(q), norm(k)
|
| 68 |
+
|
| 69 |
+
# KV Cache logic
|
| 70 |
+
if kv_cache is not None:
|
| 71 |
+
if position_id == 0:
|
| 72 |
+
# First token: initialize cache
|
| 73 |
+
kv_cache["k"] = k
|
| 74 |
+
kv_cache["v"] = v
|
| 75 |
+
else:
|
| 76 |
+
# Subsequent tokens: append to cache
|
| 77 |
+
kv_cache["k"] = torch.cat([kv_cache["k"], k], dim=1)
|
| 78 |
+
kv_cache["v"] = torch.cat([kv_cache["v"], v], dim=1)
|
| 79 |
+
|
| 80 |
+
# Use the full cached history for attention
|
| 81 |
+
k = kv_cache["k"]
|
| 82 |
+
v = kv_cache["v"]
|
| 83 |
+
|
| 84 |
nr = self.n_head // self.n_kv_head
|
| 85 |
if nr > 1:
|
| 86 |
+
k = k.unsqueeze(3).expand(B, k.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, k.size(1), self.n_head, self.head_dim)
|
| 87 |
+
v = v.unsqueeze(3).expand(B, v.size(1), self.n_kv_head, nr, self.head_dim).reshape(B, v.size(1), self.n_head, self.head_dim)
|
| 88 |
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
| 89 |
wl = window_size[0]
|
| 90 |
+
|
| 91 |
if 0 < wl < T:
|
| 92 |
+
# We only need this branch if T is large, and we're not just predicting one token.
|
| 93 |
+
# Local attention is mostly useful during pre-fill.
|
| 94 |
+
ri, ci = torch.arange(T, device=x.device).unsqueeze(1) + position_id, torch.arange(k.size(2), device=x.device).unsqueeze(0)
|
| 95 |
y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
|
| 96 |
else:
|
| 97 |
+
y = F.scaled_dot_product_attention(q, k, v, is_causal=(T > 1))
|
| 98 |
return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
|
| 99 |
|
| 100 |
class MLP(nn.Module):
|
|
|
|
| 110 |
super().__init__()
|
| 111 |
self.attn = CausalSelfAttention(config, li)
|
| 112 |
self.mlp = MLP(config)
|
| 113 |
+
def forward(self, x, ve, cs, ws, kv_cache=None, position_id=0):
|
| 114 |
+
x = x + self.attn(norm(x), ve, cs, ws, kv_cache, position_id)
|
| 115 |
x = x + self.mlp(norm(x))
|
| 116 |
return x
|
| 117 |
|
|
|
|
| 141 |
self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
|
| 142 |
self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
|
| 143 |
|
| 144 |
+
def forward(self, idx, kv_cache=None, position_id=0):
|
| 145 |
B, T = idx.size()
|
| 146 |
+
# Adjusted cos/sin slicing based on position
|
| 147 |
+
cs = self.cos[:, position_id:position_id+T], self.sin[:, position_id:position_id+T]
|
| 148 |
x = norm(self.transformer.wte(idx)); x0 = x
|
| 149 |
for i, b in enumerate(self.transformer.h):
|
| 150 |
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
|
| 151 |
ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
|
| 152 |
+
|
| 153 |
+
# Extract layer-specific cache or create one if it doesn't exist
|
| 154 |
+
layer_cache = kv_cache[i] if kv_cache is not None else None
|
| 155 |
+
|
| 156 |
+
x = b(x, ve, cs, self.window_sizes[i], layer_cache, position_id)
|
| 157 |
logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
|
| 158 |
return 15 * torch.tanh(logits / 15)
|
| 159 |
|
|
|
|
| 161 |
def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
|
| 162 |
dev = next(self.parameters()).device
|
| 163 |
ids = torch.tensor([tokens], dtype=torch.long, device=dev)
|
| 164 |
+
|
| 165 |
+
# Initialize KV Cache list: one dict per layer
|
| 166 |
+
kv_cache = [{} for _ in range(self.config.n_layer)]
|
| 167 |
+
position_id = 0
|
| 168 |
+
|
| 169 |
for _ in range(max_tokens):
|
| 170 |
+
# If position_id == 0, pass all tokens (pre-fill)
|
| 171 |
+
# Otherwise, pass ONLY the last token and rely on the cache
|
| 172 |
+
curr_ids = ids if position_id == 0 else ids[:, -1:]
|
| 173 |
+
|
| 174 |
+
# Ensure we don't go past maximum sequence length
|
| 175 |
+
if curr_ids.size(1) + position_id > self.config.sequence_len:
|
| 176 |
+
# Cache eviction logic is missing here for simplicity.
|
| 177 |
+
# In a robust implementation, you'd slice the cache to sequence_len - 1
|
| 178 |
+
curr_ids = curr_ids[:, -self.config.sequence_len:]
|
| 179 |
+
kv_cache = None # Fallback to no cache if we exceed window fully
|
| 180 |
+
position_id = 0
|
| 181 |
+
|
| 182 |
+
logits = self.forward(curr_ids, kv_cache=kv_cache, position_id=position_id)[:, -1, :]
|
| 183 |
+
|
| 184 |
if top_k > 0:
|
| 185 |
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
|
| 186 |
logits[logits < v[:, [-1]]] = -float('Inf')
|
|
|
|
| 188 |
nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
|
| 189 |
else:
|
| 190 |
nid = torch.argmax(logits, dim=-1, keepdim=True)
|
| 191 |
+
|
| 192 |
ids = torch.cat((ids, nid), dim=1)
|
| 193 |
+
|
| 194 |
+
# Update position_id for the next token
|
| 195 |
+
position_id += curr_ids.size(1)
|
| 196 |
+
if kv_cache is None:
|
| 197 |
+
position_id = 0
|
| 198 |
+
|
| 199 |
yield nid.item()
|
| 200 |
|
| 201 |
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 302 |
response = enc.decode(generated)
|
| 303 |
yield response
|
| 304 |
|
| 305 |
+
css = """
|
| 306 |
+
body {
|
| 307 |
+
background-color: #0b0f19;
|
| 308 |
+
color: #e2e8f0;
|
| 309 |
+
}
|
| 310 |
+
.gradio-container {
|
| 311 |
+
max-width: 900px !important;
|
| 312 |
+
border: 1px solid #1e293b;
|
| 313 |
+
border-radius: 12px;
|
| 314 |
+
box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.5), 0 2px 4px -1px rgba(0, 0, 0, 0.3);
|
| 315 |
+
overflow: hidden;
|
| 316 |
+
}
|
| 317 |
+
header, footer {
|
| 318 |
+
display: none !important;
|
| 319 |
+
}
|
| 320 |
+
.message-wrap {
|
| 321 |
+
border-radius: 8px;
|
| 322 |
+
margin-bottom: 8px;
|
| 323 |
+
}
|
| 324 |
+
.message-row.user-row {
|
| 325 |
+
justify-content: flex-end;
|
| 326 |
+
}
|
| 327 |
+
.message-row.bot-row {
|
| 328 |
+
justify-content: flex-start;
|
| 329 |
+
}
|
| 330 |
+
.message {
|
| 331 |
+
padding: 12px 16px;
|
| 332 |
+
border-radius: 12px;
|
| 333 |
+
font-size: 15px;
|
| 334 |
+
line-height: 1.5;
|
| 335 |
+
box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.05);
|
| 336 |
+
}
|
| 337 |
+
.message.user {
|
| 338 |
+
background-color: #1e293b;
|
| 339 |
+
color: #f8fafc;
|
| 340 |
+
border-bottom-right-radius: 4px;
|
| 341 |
+
}
|
| 342 |
+
.message.bot {
|
| 343 |
+
background-color: #0f172a;
|
| 344 |
+
color: #cbd5e1;
|
| 345 |
+
border-bottom-left-radius: 4px;
|
| 346 |
+
}
|
| 347 |
+
textarea {
|
| 348 |
+
background-color: #0f172a !important;
|
| 349 |
+
border: 1px solid #334155 !important;
|
| 350 |
+
color: #f8fafc !important;
|
| 351 |
+
border-radius: 12px !important;
|
| 352 |
+
}
|
| 353 |
+
textarea:focus {
|
| 354 |
+
border-color: #3b82f6 !important;
|
| 355 |
+
box-shadow: 0 0 0 1px #3b82f6 !important;
|
| 356 |
+
}
|
| 357 |
+
button.primary {
|
| 358 |
+
background: linear-gradient(135deg, #3b82f6, #2563eb);
|
| 359 |
+
color: white !important;
|
| 360 |
+
border-radius: 8px !important;
|
| 361 |
+
border: none !important;
|
| 362 |
+
}
|
| 363 |
+
.panel {
|
| 364 |
+
background-color: #0b0f19;
|
| 365 |
+
border: none !important;
|
| 366 |
+
}
|
| 367 |
+
"""
|
| 368 |
+
|
| 369 |
+
theme = gr.themes.Base(
|
| 370 |
+
primary_hue="blue",
|
| 371 |
+
secondary_hue="slate",
|
| 372 |
+
neutral_hue="slate",
|
| 373 |
+
spacing_size="sm",
|
| 374 |
+
radius_size="lg",
|
| 375 |
+
font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
|
| 376 |
+
).set(
|
| 377 |
+
body_background_fill="#0b0f19",
|
| 378 |
+
body_text_color="#f8fafc",
|
| 379 |
+
background_fill_primary="#0b0f19",
|
| 380 |
+
background_fill_secondary="#0f172a",
|
| 381 |
+
border_color_primary="#1e293b",
|
| 382 |
+
color_accent="#3b82f6",
|
| 383 |
+
button_primary_background_fill="*color_accent",
|
| 384 |
+
button_primary_background_fill_hover="#2563eb",
|
| 385 |
+
block_background_fill="#0f172a",
|
| 386 |
+
block_border_width="1px",
|
| 387 |
+
block_border_color="#1e293b",
|
| 388 |
+
input_background_fill="#0f172a",
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
demo = gr.ChatInterface(
|
| 392 |
respond,
|
| 393 |
type="messages",
|
| 394 |
+
title="Treevik-2 (2.5B)",
|
| 395 |
+
description="""
|
| 396 |
+
<div style="text-align: center; margin-top: -10px; margin-bottom: 20px;">
|
| 397 |
+
<p style="color: #64748b; font-size: 14px; margin: 0; font-weight: 500; letter-spacing: 0.5px; text-transform: uppercase;">PlusB Intelligence Lab</p>
|
| 398 |
+
<p style="color: #475569; font-size: 12px; margin-top: 4px;">Instruction-tuned 2.5B Parameter Model β’ 8-bit Quantized</p>
|
| 399 |
+
</div>
|
| 400 |
+
""",
|
| 401 |
+
theme=theme,
|
| 402 |
+
css=css,
|
| 403 |
+
chatbot=gr.Chatbot(height=550, show_label=False, show_copy_button=True, avatar_images=(None, "https://ui-avatars.com/api/?name=PB&background=2563eb&color=fff&size=256&font-size=0.4&bold=true")),
|
| 404 |
+
textbox=gr.Textbox(placeholder="Ask Treevik anything...", container=False, scale=7, min_lines=1, max_lines=5),
|
| 405 |
+
submit_btn=gr.Button("Send", variant="primary", scale=1),
|
| 406 |
+
retry_btn=None,
|
| 407 |
+
undo_btn=None,
|
| 408 |
+
clear_btn=gr.Button("Clear Chat", variant="secondary", size="sm"),
|
| 409 |
)
|
| 410 |
|
| 411 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|