treevik1-chat / app.py
PlusBIntelligenceLab's picture
trying ....
031335e verified
Raw History Blame Contribute Delete
11 kB
"""
Treevik-1 500M Instruct β€” HuggingFace Space
Downloads model from HF Hub: PlusBIntelligenceLab/Treevik1-500M-Instruct
"""
import os
import json
import pickle
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
import gradio as gr
from huggingface_hub import hf_hub_download
# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Download Model from HuggingFace Hub β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
REPO_ID = "PlusBIntelligenceLab/Treevik1-500M-Instruct"
print(f"Downloading model from {REPO_ID}...")
meta_path = hf_hub_download(REPO_ID, "meta_sft.json")
weights_path = hf_hub_download(REPO_ID, "treevik1-500M-Instruct.pt")
tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl")
print("βœ… Downloaded!")
# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Model Architecture β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
def norm(x):
return F.rms_norm(x, (x.size(-1),))
def has_ve(li, nl):
return li % 2 == (nl - 1) % 2
def apply_rotary_emb(x, cos, sin):
d = x.shape[3] // 2
x1, x2 = x[..., :d], x[..., d:]
return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3)
class CausalSelfAttention(nn.Module):
def __init__(self, config, layer_idx):
super().__init__()
self.n_head, self.n_kv_head, self.n_embd = config.n_head, config.n_kv_head, config.n_embd
self.head_dim = self.n_embd // self.n_head
self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False)
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None
def forward(self, x, ve, cos_sin, window_size):
B, T, C = x.size()
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim)
if ve is not None:
ve = ve.view(B, T, self.n_kv_head, self.head_dim)
v = v + (2 * torch.sigmoid(self.ve_gate(x[..., :32]))).unsqueeze(-1) * ve
cos, sin = cos_sin
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
q, k = norm(q), norm(k)
nr = self.n_head // self.n_kv_head
if nr > 1:
k = k.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
v = v.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim)
q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
wl = window_size[0]
if 0 < wl < T:
ri, ci = torch.arange(T, device=x.device).unsqueeze(1), torch.arange(T, device=x.device).unsqueeze(0)
y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype))
else:
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1))
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
def forward(self, x):
return self.c_proj(F.relu(self.c_fc(x)).square())
class Block(nn.Module):
def __init__(self, config, li):
super().__init__()
self.attn = CausalSelfAttention(config, li)
self.mlp = MLP(config)
def forward(self, x, ve, cs, ws):
x = x + self.attn(norm(x), ve, cs, ws)
x = x + self.mlp(norm(x))
return x
@dataclass
class GPTConfig:
sequence_len: int = 2048; vocab_size: int = 65536; n_layer: int = 12
n_head: int = 6; n_kv_head: int = 6; n_embd: int = 768; window_pattern: str = "SSSL"
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
pat = config.window_pattern.upper()
lw, sw = config.sequence_len, config.sequence_len // 2
self.window_sizes = [((sw, 0) if c == 'S' else (lw, 0)) for i, c in enumerate((pat * 4)[:config.n_layer])]
self.window_sizes[-1] = (lw, 0)
pad = ((config.vocab_size + 63) // 64) * 64
self.transformer = nn.ModuleDict({"wte": nn.Embedding(pad, config.n_embd), "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)])})
self.lm_head = nn.Linear(config.n_embd, pad, bias=False)
self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))
self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))
hd = config.n_embd // config.n_head
self.value_embeds = nn.ModuleDict({str(i): nn.Embedding(pad, config.n_kv_head * hd) for i in range(config.n_layer) if has_ve(i, config.n_layer)})
self.rotary_seq_len = config.sequence_len * 10
inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd))
freqs = torch.outer(torch.arange(self.rotary_seq_len, dtype=torch.float32), inv)
self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False)
self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False)
def forward(self, idx):
B, T = idx.size()
cs = self.cos[:, :T], self.sin[:, :T]
x = norm(self.transformer.wte(idx)); x0 = x
for i, b in enumerate(self.transformer.h):
x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None
x = b(x, ve, cs, self.window_sizes[i])
logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float()
return 15 * torch.tanh(logits / 15)
@torch.inference_mode()
def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50):
dev = next(self.parameters()).device
ids = torch.tensor([tokens], dtype=torch.long, device=dev)
for _ in range(max_tokens):
logits = self.forward(ids[:, -self.config.sequence_len:])[:, -1, :]
if top_k > 0:
v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
logits[logits < v[:, [-1]]] = -float('Inf')
if temperature > 0:
nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1)
else:
nid = torch.argmax(logits, dim=-1, keepdim=True)
ids = torch.cat((ids, nid), dim=1)
yield nid.item()
# ╔══════════════════════════════════════════════════════════════════════════╗
# β•‘ Load Model β•‘
# β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•
meta = json.load(open(meta_path))
mc = meta["model_config"]
config = GPTConfig(**{k: mc[k] for k in ['sequence_len','vocab_size','n_layer','n_head','n_kv_head','n_embd']})
config.window_pattern = mc.get("window_pattern", "SSSL")
device = "cuda" if torch.cuda.is_available() else "cpu"
model = GPT(config)
model.load_state_dict(torch.load(weights_path, map_location="cpu", weights_only=True))
model = model.to(device=device, dtype=torch.float32).eval()
if device == "cuda":
hd = config.n_embd // config.n_head
inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32, device=device) / hd))
freqs = torch.outer(torch.arange(model.rotary_seq_len, dtype=torch.float32, device=device), inv)
model.cos = freqs.cos()[None, :, None, :]
model.sin = freqs.sin()[None, :, None, :]
with open(tokenizer_path, "rb") as f:
enc = pickle.load(f)
BOS = enc.encode_single_token("<|bos|>")
USER_START = enc.encode_single_token("<|user_start|>")
USER_END = enc.encode_single_token("<|user_end|>")
ASST_START = enc.encode_single_token("<|assistant_start|>")
ASST_END = enc.encode_single_token("<|assistant_end|>")
print(f"βœ… Loaded on {device} ({sum(p.numel() for p in model.parameters())/1e6:.0f}M params)")
def respond(message, chat_history):
if not message.strip():
yield ""
return
# Build prompt from messages-format history
ids = [BOS]
for entry in chat_history:
role, content = entry["role"], entry["content"]
if role == "user":
ids += [USER_START] + enc.encode_ordinary(content) + [USER_END]
elif role == "assistant":
ids += [ASST_START] + enc.encode_ordinary(content) + [ASST_END]
ids += [USER_START] + enc.encode_ordinary(message) + [USER_END]
ids.append(ASST_START)
# Generate full response
generated = []
for token in model.generate(ids, max_tokens=512, temperature=0.7, top_k=50):
if token == ASST_END:
break
generated.append(token)
response = enc.decode(generated)
yield response
demo = gr.ChatInterface(
respond,
type="messages",
title="🌳 Treevik-1 500M Instruct",
description="**By PlusB Intelligence Lab** β€’ Created by Sayan Chowdhury\n\n*500M parameter model β€” may produce incorrect information.*",
theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue"),
chatbot=gr.Chatbot(height=500),
textbox=gr.Textbox(placeholder="Type a message and press Enter...", container=False, scale=7),
)
demo.launch(server_name="0.0.0.0", server_port=7860)