treevik2-chat / app.py
PlusBIntelligenceLab's picture
Update app.py
a7ca58f verified
Raw History Blame Contribute Delete
15.9 kB
"""
Treevik-2 2.5B Instruct
HF Spaces Ready
GPU + CPU Auto Detect
Modern ChatGPT Style UI
"""
import gc
import json
import pickle
from dataclasses import dataclass
import gradio as gr
import spaces
import torch
import torch.nn as nn
import torch.nn.functional as F
from huggingface_hub import hf_hub_download
# ============================================================================
# DEVICE
# ============================================================================
torch.set_grad_enabled(False)
if torch.cuda.is_available():
device = "cuda"
dtype = torch.float16
torch.backends.cuda.matmul.allow_tf32 = True
else:
device = "cpu"
dtype = torch.float32
print(f"Using device: {device}")
print(f"Using dtype: {dtype}")
# ============================================================================
# DOWNLOAD MODEL
# ============================================================================
REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct"
print(f"Downloading model from {REPO_ID}...")
meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt")
tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
# ============================================================================
# MODEL ARCHITECTURE
# ============================================================================
def norm(x):
return F.rms_norm(x, (x.size(-1),))
def has_ve(li, nl):
return li % 2 == (nl - 1) % 2
def apply_rotary_emb(x, cos, sin):
d = x.shape[3] // 2
x1, x2 = x[..., :d], x[..., d:]
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3)
class CausalSelfAttention(nn.Module):
def __init__(self, config, layer_idx):
super().__init__()
self.n_head = config.n_head
self.n_kv_head = config.n_kv_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
self.c_q = nn.Linear(
self.n_embd,
self.n_head * self.head_dim,
bias=False,
)
self.c_k = nn.Linear(
self.n_embd,
self.n_kv_head * self.head_dim,
bias=False,
)
self.c_v = nn.Linear(
self.n_embd,
self.n_kv_head * self.head_dim,
bias=False,
)
self.c_proj = nn.Linear(
self.n_embd,
self.n_embd,
bias=False,
)
self.ve_gate = (
nn.Linear(32, self.n_kv_head, bias=False)
if has_ve(layer_idx, config.n_layer)
else None
)
def forward(
self,
x,
ve,
cos_sin,
window_size,
kv_cache=None,
position_id=0,
):
B, T, C = x.size()
q = self.c_q(x).view(
B, T, self.n_head, self.head_dim
)
k = self.c_k(x).view(
B, T, self.n_kv_head, self.head_dim
)
v = self.c_v(x).view(
B, T, self.n_kv_head, self.head_dim
)
if ve is not None:
ve = ve.view(
B,
T,
self.n_kv_head,
self.head_dim,
)
v = v + (
2 * torch.sigmoid(
self.ve_gate(x[..., :32])
)
).unsqueeze(-1) * ve
cos, sin = cos_sin
q = apply_rotary_emb(q, cos, sin)
k = apply_rotary_emb(k, cos, sin)
q = norm(q)
k = norm(k)
if kv_cache is not None:
if position_id == 0:
kv_cache["k"] = k
kv_cache["v"] = v
else:
kv_cache["k"] = torch.cat(
[kv_cache["k"], k],
dim=1,
)
kv_cache["v"] = torch.cat(
[kv_cache["v"], v],
dim=1,
)
k = kv_cache["k"]
v = kv_cache["v"]
nr = self.n_head // self.n_kv_head
if nr > 1:
k = (
k.unsqueeze(3)
.expand(
B,
k.size(1),
self.n_kv_head,
nr,
self.head_dim,
)
.reshape(
B,
k.size(1),
self.n_head,
self.head_dim,
)
)
v = (
v.unsqueeze(3)
.expand(
B,
v.size(1),
self.n_kv_head,
nr,
self.head_dim,
)
.reshape(
B,
v.size(1),
self.n_head,
self.head_dim,
)
)
q = q.transpose(1, 2)
k = k.transpose(1, 2)
v = v.transpose(1, 2)
y = F.scaled_dot_product_attention(
q,
k,
v,
is_causal=(T > 1),
)
y = (
y.transpose(1, 2)
.contiguous()
.view(B, T, -1)
)
return self.c_proj(y)
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(
config.n_embd,
4 * config.n_embd,
bias=False,
)
self.c_proj = nn.Linear(
4 * config.n_embd,
config.n_embd,
bias=False,
)
def forward(self, x):
return self.c_proj(
F.relu(self.c_fc(x)).square()
)
class Block(nn.Module):
def __init__(self, config, li):
super().__init__()
self.attn = CausalSelfAttention(config, li)
self.mlp = MLP(config)
def forward(
self,
x,
ve,
cs,
ws,
kv_cache=None,
position_id=0,
):
x = x + self.attn(
norm(x),
ve,
cs,
ws,
kv_cache,
position_id,
)
x = x + self.mlp(norm(x))
return x
@dataclass
class GPTConfig:
sequence_len: int = 2048
vocab_size: int = 65536
n_layer: int = 26
n_head: int = 13
n_kv_head: int = 13
n_embd: int = 1664
window_pattern: str = "L"
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
pat = config.window_pattern.upper()
lw = config.sequence_len
sw = config.sequence_len // 2
self.window_sizes = [
(
(sw, 0)
if c == "S"
else (lw, 0)
)
for c in (pat * config.n_layer)[:config.n_layer]
]
self.window_sizes[-1] = (lw, 0)
pad = ((config.vocab_size + 63) // 64) * 64
self.transformer = nn.ModuleDict({
"wte": nn.Embedding(
pad,
config.n_embd,
),
"h": nn.ModuleList([
Block(config, i)
for i in range(config.n_layer)
]),
})
self.lm_head = nn.Linear(
config.n_embd,
pad,
bias=False,
)
self.resid_lambdas = nn.Parameter(
torch.ones(config.n_layer)
)
self.x0_lambdas = nn.Parameter(
torch.zeros(config.n_layer)
)
hd = config.n_embd // config.n_head
self.value_embeds = nn.ModuleDict({
str(i): nn.Embedding(
pad,
config.n_kv_head * hd,
)
for i in range(config.n_layer)
if has_ve(i, config.n_layer)
})
self.rotary_seq_len = config.sequence_len * 10
inv = 1.0 / (
10000 ** (
torch.arange(
0,
hd,
2,
dtype=torch.float32,
) / hd
)
)
freqs = torch.outer(
torch.arange(
self.rotary_seq_len,
dtype=torch.float32,
),
inv,
)
self.register_buffer(
"cos",
freqs.cos()[None, :, None, :],
persistent=False,
)
self.register_buffer(
"sin",
freqs.sin()[None, :, None, :],
persistent=False,
)
def forward(
self,
idx,
kv_cache=None,
position_id=0,
):
B, T = idx.size()
cs = (
self.cos[:, position_id:position_id + T],
self.sin[:, position_id:position_id + T],
)
x = norm(
self.transformer.wte(idx)
)
x0 = x
for i, b in enumerate(self.transformer.h):
x = (
self.resid_lambdas[i] * x
+ self.x0_lambdas[i] * x0
)
ve = (
self.value_embeds[str(i)](idx)
if str(i) in self.value_embeds
else None
)
layer_cache = (
kv_cache[i]
if kv_cache is not None
else None
)
x = b(
x,
ve,
cs,
self.window_sizes[i],
layer_cache,
position_id,
)
logits = self.lm_head(norm(x))
logits = logits[..., :self.config.vocab_size]
return 15 * torch.tanh(logits / 15)
@torch.inference_mode()
def generate(
self,
tokens,
max_tokens=512,
temperature=0.7,
top_k=50,
):
ids = torch.tensor(
[tokens],
dtype=torch.long,
device=device,
)
kv_cache = [
{}
for _ in range(self.config.n_layer)
]
position_id = 0
for _ in range(max_tokens):
curr_ids = (
ids
if position_id == 0
else ids[:, -1:]
)
logits = self.forward(
curr_ids,
kv_cache=kv_cache,
position_id=position_id,
)[:, -1, :]
if top_k > 0:
v, _ = torch.topk(
logits,
min(top_k, logits.size(-1))
)
logits[
logits < v[:, [-1]]
] = -float("Inf")
if temperature > 0:
nid = torch.multinomial(
F.softmax(
logits / temperature,
dim=-1,
),
1,
)
else:
nid = torch.argmax(
logits,
dim=-1,
keepdim=True,
)
ids = torch.cat(
(ids, nid),
dim=1,
)
position_id += curr_ids.size(1)
yield nid.item()
# ============================================================================
# LOAD MODEL
# ============================================================================
meta = json.load(open(meta_path))
mc = meta["model_config"]
config = GPTConfig(**{
k: mc[k]
for k in [
"sequence_len",
"vocab_size",
"n_layer",
"n_head",
"n_kv_head",
"n_embd",
]
})
config.window_pattern = mc.get(
"window_pattern",
"L",
)
model = GPT(config)
print("Loading weights...")
raw_state_dict = torch.load(
weights_path,
map_location="cpu",
weights_only=True,
)
model_state = model.state_dict()
for key in list(model_state.keys()):
if key not in raw_state_dict:
continue
tensor = raw_state_dict[key]
if tensor.dtype == torch.int8:
scale_key = key + "_scale"
if scale_key in raw_state_dict:
tensor = (
tensor.float()
* raw_state_dict[scale_key].float()
)
model_state[key].copy_(
tensor.to(model_state[key].dtype)
)
print("Moving model to device...")
model = model.to(
device=device,
dtype=dtype,
)
model.eval()
del raw_state_dict
gc.collect()
if device == "cuda":
torch.cuda.empty_cache()
# ============================================================================
# TOKENIZER
# ============================================================================
with open(tokenizer_path, "rb") as f:
enc = pickle.load(f)
BOS = enc.encode_single_token("<|bos|>")
USER_START = enc.encode_single_token("<|user_start|>")
USER_END = enc.encode_single_token("<|user_end|>")
ASST_START = enc.encode_single_token("<|assistant_start|>")
ASST_END = enc.encode_single_token("<|assistant_end|>")
# ============================================================================
# CHAT
# ============================================================================
@spaces.GPU
def respond(message, history):
if not message.strip():
yield ""
return
ids = [BOS]
ids += [USER_START]
ids += enc.encode_ordinary(message)
ids += [USER_END]
ids += [ASST_START]
generated = []
for token in model.generate(
ids,
max_tokens=2048,
temperature=0.5,
top_k=50,
):
if token == ASST_END:
break
generated.append(token)
partial = enc.decode(generated)
yield partial
# ============================================================================
# UI
# ============================================================================
css = """
body {
background: #0f172a;
}
.gradio-container {
max-width: 1100px !important;
margin: auto;
}
footer {
display: none !important;
}
.message.user {
background: #2563eb !important;
color: white !important;
border-radius: 18px !important;
}
.message.bot {
background: #1e293b !important;
color: #f8fafc !important;
border-radius: 18px !important;
}
textarea {
background: #111827 !important;
color: white !important;
border-radius: 16px !important;
border: 1px solid #334155 !important;
}
"""
theme = gr.themes.Soft(
primary_hue="blue",
secondary_hue="slate",
neutral_hue="slate",
)
demo = gr.ChatInterface(
fn=respond,
title="Treevik-2 (2.5B)",
description="""
<div style='text-align:center'>
<h3>By PlusB Intelligence Lab</h3>
<p>Instruction Tuned 2.5B Parameter Model</p>
</div>
""",
chatbot=gr.Chatbot(
height=650,
show_copy_button=True,
bubble_full_width=False,
type="messages",
),
textbox=gr.Textbox(
placeholder="Talk to Treevik...",
container=False,
),
theme=theme,
css=css,
)
demo.queue()
demo.launch(
server_name="0.0.0.0",
server_port=7860,
)