Spaces:
Sleeping
Sleeping
Download app.py from PlusBIntelligenceLab/treevik1-chat: direct link, hf CLI and curl.
- Browser
- Download file 11 kB
-
https://huggingface.co/spaces/PlusBIntelligenceLab/treevik1-chat/resolve/main/app.py
- Command line
-
hf download hf://spaces/PlusBIntelligenceLab/treevik1-chat/app.py
-
curl -L -o app.py https://huggingface.co/spaces/PlusBIntelligenceLab/treevik1-chat/resolve/main/app.py
11 kB
| """ | |
| Treevik-1 500M Instruct β HuggingFace Space | |
| Downloads model from HF Hub: PlusBIntelligenceLab/Treevik1-500M-Instruct | |
| """ | |
| import os | |
| import json | |
| import pickle | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import gradio as gr | |
| from huggingface_hub import hf_hub_download | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β Download Model from HuggingFace Hub β | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| REPO_ID = "PlusBIntelligenceLab/Treevik1-500M-Instruct" | |
| print(f"Downloading model from {REPO_ID}...") | |
| meta_path = hf_hub_download(REPO_ID, "meta_sft.json") | |
| weights_path = hf_hub_download(REPO_ID, "treevik1-500M-Instruct.pt") | |
| tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl") | |
| print("β Downloaded!") | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β Model Architecture β | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def norm(x): | |
| return F.rms_norm(x, (x.size(-1),)) | |
| def has_ve(li, nl): | |
| return li % 2 == (nl - 1) % 2 | |
| def apply_rotary_emb(x, cos, sin): | |
| d = x.shape[3] // 2 | |
| x1, x2 = x[..., :d], x[..., d:] | |
| return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3) | |
| class CausalSelfAttention(nn.Module): | |
| def __init__(self, config, layer_idx): | |
| super().__init__() | |
| self.n_head, self.n_kv_head, self.n_embd = config.n_head, config.n_kv_head, config.n_embd | |
| self.head_dim = self.n_embd // self.n_head | |
| self.c_q = nn.Linear(self.n_embd, self.n_head * self.head_dim, bias=False) | |
| self.c_k = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False) | |
| self.c_v = nn.Linear(self.n_embd, self.n_kv_head * self.head_dim, bias=False) | |
| self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False) | |
| self.ve_gate = nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None | |
| def forward(self, x, ve, cos_sin, window_size): | |
| B, T, C = x.size() | |
| q = self.c_q(x).view(B, T, self.n_head, self.head_dim) | |
| k = self.c_k(x).view(B, T, self.n_kv_head, self.head_dim) | |
| v = self.c_v(x).view(B, T, self.n_kv_head, self.head_dim) | |
| if ve is not None: | |
| ve = ve.view(B, T, self.n_kv_head, self.head_dim) | |
| v = v + (2 * torch.sigmoid(self.ve_gate(x[..., :32]))).unsqueeze(-1) * ve | |
| cos, sin = cos_sin | |
| q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin) | |
| q, k = norm(q), norm(k) | |
| nr = self.n_head // self.n_kv_head | |
| if nr > 1: | |
| k = k.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim) | |
| v = v.unsqueeze(3).expand(B, T, self.n_kv_head, nr, self.head_dim).reshape(B, T, self.n_head, self.head_dim) | |
| q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) | |
| wl = window_size[0] | |
| if 0 < wl < T: | |
| ri, ci = torch.arange(T, device=x.device).unsqueeze(1), torch.arange(T, device=x.device).unsqueeze(0) | |
| y = F.scaled_dot_product_attention(q, k, v, attn_mask=torch.where((ci <= ri) & ((ri - ci) < wl), 0.0, float('-inf')).to(q.dtype)) | |
| else: | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, -1)) | |
| class MLP(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False) | |
| self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False) | |
| def forward(self, x): | |
| return self.c_proj(F.relu(self.c_fc(x)).square()) | |
| class Block(nn.Module): | |
| def __init__(self, config, li): | |
| super().__init__() | |
| self.attn = CausalSelfAttention(config, li) | |
| self.mlp = MLP(config) | |
| def forward(self, x, ve, cs, ws): | |
| x = x + self.attn(norm(x), ve, cs, ws) | |
| x = x + self.mlp(norm(x)) | |
| return x | |
| class GPTConfig: | |
| sequence_len: int = 2048; vocab_size: int = 65536; n_layer: int = 12 | |
| n_head: int = 6; n_kv_head: int = 6; n_embd: int = 768; window_pattern: str = "SSSL" | |
| class GPT(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.config = config | |
| pat = config.window_pattern.upper() | |
| lw, sw = config.sequence_len, config.sequence_len // 2 | |
| self.window_sizes = [((sw, 0) if c == 'S' else (lw, 0)) for i, c in enumerate((pat * 4)[:config.n_layer])] | |
| self.window_sizes[-1] = (lw, 0) | |
| pad = ((config.vocab_size + 63) // 64) * 64 | |
| self.transformer = nn.ModuleDict({"wte": nn.Embedding(pad, config.n_embd), "h": nn.ModuleList([Block(config, i) for i in range(config.n_layer)])}) | |
| self.lm_head = nn.Linear(config.n_embd, pad, bias=False) | |
| self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) | |
| self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) | |
| hd = config.n_embd // config.n_head | |
| self.value_embeds = nn.ModuleDict({str(i): nn.Embedding(pad, config.n_kv_head * hd) for i in range(config.n_layer) if has_ve(i, config.n_layer)}) | |
| self.rotary_seq_len = config.sequence_len * 10 | |
| inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32) / hd)) | |
| freqs = torch.outer(torch.arange(self.rotary_seq_len, dtype=torch.float32), inv) | |
| self.register_buffer("cos", freqs.cos()[None, :, None, :], persistent=False) | |
| self.register_buffer("sin", freqs.sin()[None, :, None, :], persistent=False) | |
| def forward(self, idx): | |
| B, T = idx.size() | |
| cs = self.cos[:, :T], self.sin[:, :T] | |
| x = norm(self.transformer.wte(idx)); x0 = x | |
| for i, b in enumerate(self.transformer.h): | |
| x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 | |
| ve = self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None | |
| x = b(x, ve, cs, self.window_sizes[i]) | |
| logits = self.lm_head(norm(x))[..., :self.config.vocab_size].float() | |
| return 15 * torch.tanh(logits / 15) | |
| def generate(self, tokens, max_tokens=512, temperature=0.7, top_k=50): | |
| dev = next(self.parameters()).device | |
| ids = torch.tensor([tokens], dtype=torch.long, device=dev) | |
| for _ in range(max_tokens): | |
| logits = self.forward(ids[:, -self.config.sequence_len:])[:, -1, :] | |
| if top_k > 0: | |
| v, _ = torch.topk(logits, min(top_k, logits.size(-1))) | |
| logits[logits < v[:, [-1]]] = -float('Inf') | |
| if temperature > 0: | |
| nid = torch.multinomial(F.softmax(logits / temperature, dim=-1), 1) | |
| else: | |
| nid = torch.argmax(logits, dim=-1, keepdim=True) | |
| ids = torch.cat((ids, nid), dim=1) | |
| yield nid.item() | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| # β Load Model β | |
| # ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| meta = json.load(open(meta_path)) | |
| mc = meta["model_config"] | |
| config = GPTConfig(**{k: mc[k] for k in ['sequence_len','vocab_size','n_layer','n_head','n_kv_head','n_embd']}) | |
| config.window_pattern = mc.get("window_pattern", "SSSL") | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model = GPT(config) | |
| model.load_state_dict(torch.load(weights_path, map_location="cpu", weights_only=True)) | |
| model = model.to(device=device, dtype=torch.float32).eval() | |
| if device == "cuda": | |
| hd = config.n_embd // config.n_head | |
| inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, dtype=torch.float32, device=device) / hd)) | |
| freqs = torch.outer(torch.arange(model.rotary_seq_len, dtype=torch.float32, device=device), inv) | |
| model.cos = freqs.cos()[None, :, None, :] | |
| model.sin = freqs.sin()[None, :, None, :] | |
| with open(tokenizer_path, "rb") as f: | |
| enc = pickle.load(f) | |
| BOS = enc.encode_single_token("<|bos|>") | |
| USER_START = enc.encode_single_token("<|user_start|>") | |
| USER_END = enc.encode_single_token("<|user_end|>") | |
| ASST_START = enc.encode_single_token("<|assistant_start|>") | |
| ASST_END = enc.encode_single_token("<|assistant_end|>") | |
| print(f"β Loaded on {device} ({sum(p.numel() for p in model.parameters())/1e6:.0f}M params)") | |
| def respond(message, chat_history): | |
| if not message.strip(): | |
| yield "" | |
| return | |
| # Build prompt from messages-format history | |
| ids = [BOS] | |
| for entry in chat_history: | |
| role, content = entry["role"], entry["content"] | |
| if role == "user": | |
| ids += [USER_START] + enc.encode_ordinary(content) + [USER_END] | |
| elif role == "assistant": | |
| ids += [ASST_START] + enc.encode_ordinary(content) + [ASST_END] | |
| ids += [USER_START] + enc.encode_ordinary(message) + [USER_END] | |
| ids.append(ASST_START) | |
| # Generate full response | |
| generated = [] | |
| for token in model.generate(ids, max_tokens=512, temperature=0.7, top_k=50): | |
| if token == ASST_END: | |
| break | |
| generated.append(token) | |
| response = enc.decode(generated) | |
| yield response | |
| demo = gr.ChatInterface( | |
| respond, | |
| type="messages", | |
| title="π³ Treevik-1 500M Instruct", | |
| description="**By PlusB Intelligence Lab** β’ Created by Sayan Chowdhury\n\n*500M parameter model β may produce incorrect information.*", | |
| theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue"), | |
| chatbot=gr.Chatbot(height=500), | |
| textbox=gr.Textbox(placeholder="Type a message and press Enter...", container=False, scale=7), | |
| ) | |
| demo.launch(server_name="0.0.0.0", server_port=7860) | |