Download app.py from PlusBIntelligenceLab/treevik2-chat: direct link, hf CLI and curl.
- Browser
- Download file 15.9 kB
-
https://huggingface.co/spaces/PlusBIntelligenceLab/treevik2-chat/resolve/main/app.py
- Command line
-
hf download hf://spaces/PlusBIntelligenceLab/treevik2-chat/app.py
-
curl -L -o app.py https://huggingface.co/spaces/PlusBIntelligenceLab/treevik2-chat/resolve/main/app.py
15.9 kB
| """ | |
| Treevik-2 2.5B Instruct | |
| HF Spaces Ready | |
| GPU + CPU Auto Detect | |
| Modern ChatGPT Style UI | |
| """ | |
| import gc | |
| import json | |
| import pickle | |
| from dataclasses import dataclass | |
| import gradio as gr | |
| import spaces | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from huggingface_hub import hf_hub_download | |
| # ============================================================================ | |
| # DEVICE | |
| # ============================================================================ | |
| torch.set_grad_enabled(False) | |
| if torch.cuda.is_available(): | |
| device = "cuda" | |
| dtype = torch.float16 | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| else: | |
| device = "cpu" | |
| dtype = torch.float32 | |
| print(f"Using device: {device}") | |
| print(f"Using dtype: {dtype}") | |
| # ============================================================================ | |
| # DOWNLOAD MODEL | |
| # ============================================================================ | |
| REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct" | |
| print(f"Downloading model from {REPO_ID}...") | |
| meta_path = hf_hub_download(REPO_ID, "meta_sft.json") | |
| weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt") | |
| tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl") | |
| # ============================================================================ | |
| # MODEL ARCHITECTURE | |
| # ============================================================================ | |
| def norm(x): | |
| return F.rms_norm(x, (x.size(-1),)) | |
| def has_ve(li, nl): | |
| return li % 2 == (nl - 1) % 2 | |
| def apply_rotary_emb(x, cos, sin): | |
| d = x.shape[3] // 2 | |
| x1, x2 = x[..., :d], x[..., d:] | |
| return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3) | |
| class CausalSelfAttention(nn.Module): | |
| def __init__(self, config, layer_idx): | |
| super().__init__() | |
| self.n_head = config.n_head | |
| self.n_kv_head = config.n_kv_head | |
| self.n_embd = config.n_embd | |
| self.head_dim = self.n_embd // self.n_head | |
| self.c_q = nn.Linear( | |
| self.n_embd, | |
| self.n_head * self.head_dim, | |
| bias=False, | |
| ) | |
| self.c_k = nn.Linear( | |
| self.n_embd, | |
| self.n_kv_head * self.head_dim, | |
| bias=False, | |
| ) | |
| self.c_v = nn.Linear( | |
| self.n_embd, | |
| self.n_kv_head * self.head_dim, | |
| bias=False, | |
| ) | |
| self.c_proj = nn.Linear( | |
| self.n_embd, | |
| self.n_embd, | |
| bias=False, | |
| ) | |
| self.ve_gate = ( | |
| nn.Linear(32, self.n_kv_head, bias=False) | |
| if has_ve(layer_idx, config.n_layer) | |
| else None | |
| ) | |
| def forward( | |
| self, | |
| x, | |
| ve, | |
| cos_sin, | |
| window_size, | |
| kv_cache=None, | |
| position_id=0, | |
| ): | |
| B, T, C = x.size() | |
| q = self.c_q(x).view( | |
| B, T, self.n_head, self.head_dim | |
| ) | |
| k = self.c_k(x).view( | |
| B, T, self.n_kv_head, self.head_dim | |
| ) | |
| v = self.c_v(x).view( | |
| B, T, self.n_kv_head, self.head_dim | |
| ) | |
| if ve is not None: | |
| ve = ve.view( | |
| B, | |
| T, | |
| self.n_kv_head, | |
| self.head_dim, | |
| ) | |
| v = v + ( | |
| 2 * torch.sigmoid( | |
| self.ve_gate(x[..., :32]) | |
| ) | |
| ).unsqueeze(-1) * ve | |
| cos, sin = cos_sin | |
| q = apply_rotary_emb(q, cos, sin) | |
| k = apply_rotary_emb(k, cos, sin) | |
| q = norm(q) | |
| k = norm(k) | |
| if kv_cache is not None: | |
| if position_id == 0: | |
| kv_cache["k"] = k | |
| kv_cache["v"] = v | |
| else: | |
| kv_cache["k"] = torch.cat( | |
| [kv_cache["k"], k], | |
| dim=1, | |
| ) | |
| kv_cache["v"] = torch.cat( | |
| [kv_cache["v"], v], | |
| dim=1, | |
| ) | |
| k = kv_cache["k"] | |
| v = kv_cache["v"] | |
| nr = self.n_head // self.n_kv_head | |
| if nr > 1: | |
| k = ( | |
| k.unsqueeze(3) | |
| .expand( | |
| B, | |
| k.size(1), | |
| self.n_kv_head, | |
| nr, | |
| self.head_dim, | |
| ) | |
| .reshape( | |
| B, | |
| k.size(1), | |
| self.n_head, | |
| self.head_dim, | |
| ) | |
| ) | |
| v = ( | |
| v.unsqueeze(3) | |
| .expand( | |
| B, | |
| v.size(1), | |
| self.n_kv_head, | |
| nr, | |
| self.head_dim, | |
| ) | |
| .reshape( | |
| B, | |
| v.size(1), | |
| self.n_head, | |
| self.head_dim, | |
| ) | |
| ) | |
| q = q.transpose(1, 2) | |
| k = k.transpose(1, 2) | |
| v = v.transpose(1, 2) | |
| y = F.scaled_dot_product_attention( | |
| q, | |
| k, | |
| v, | |
| is_causal=(T > 1), | |
| ) | |
| y = ( | |
| y.transpose(1, 2) | |
| .contiguous() | |
| .view(B, T, -1) | |
| ) | |
| return self.c_proj(y) | |
| class MLP(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.c_fc = nn.Linear( | |
| config.n_embd, | |
| 4 * config.n_embd, | |
| bias=False, | |
| ) | |
| self.c_proj = nn.Linear( | |
| 4 * config.n_embd, | |
| config.n_embd, | |
| bias=False, | |
| ) | |
| def forward(self, x): | |
| return self.c_proj( | |
| F.relu(self.c_fc(x)).square() | |
| ) | |
| class Block(nn.Module): | |
| def __init__(self, config, li): | |
| super().__init__() | |
| self.attn = CausalSelfAttention(config, li) | |
| self.mlp = MLP(config) | |
| def forward( | |
| self, | |
| x, | |
| ve, | |
| cs, | |
| ws, | |
| kv_cache=None, | |
| position_id=0, | |
| ): | |
| x = x + self.attn( | |
| norm(x), | |
| ve, | |
| cs, | |
| ws, | |
| kv_cache, | |
| position_id, | |
| ) | |
| x = x + self.mlp(norm(x)) | |
| return x | |
| class GPTConfig: | |
| sequence_len: int = 2048 | |
| vocab_size: int = 65536 | |
| n_layer: int = 26 | |
| n_head: int = 13 | |
| n_kv_head: int = 13 | |
| n_embd: int = 1664 | |
| window_pattern: str = "L" | |
| class GPT(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| self.config = config | |
| pat = config.window_pattern.upper() | |
| lw = config.sequence_len | |
| sw = config.sequence_len // 2 | |
| self.window_sizes = [ | |
| ( | |
| (sw, 0) | |
| if c == "S" | |
| else (lw, 0) | |
| ) | |
| for c in (pat * config.n_layer)[:config.n_layer] | |
| ] | |
| self.window_sizes[-1] = (lw, 0) | |
| pad = ((config.vocab_size + 63) // 64) * 64 | |
| self.transformer = nn.ModuleDict({ | |
| "wte": nn.Embedding( | |
| pad, | |
| config.n_embd, | |
| ), | |
| "h": nn.ModuleList([ | |
| Block(config, i) | |
| for i in range(config.n_layer) | |
| ]), | |
| }) | |
| self.lm_head = nn.Linear( | |
| config.n_embd, | |
| pad, | |
| bias=False, | |
| ) | |
| self.resid_lambdas = nn.Parameter( | |
| torch.ones(config.n_layer) | |
| ) | |
| self.x0_lambdas = nn.Parameter( | |
| torch.zeros(config.n_layer) | |
| ) | |
| hd = config.n_embd // config.n_head | |
| self.value_embeds = nn.ModuleDict({ | |
| str(i): nn.Embedding( | |
| pad, | |
| config.n_kv_head * hd, | |
| ) | |
| for i in range(config.n_layer) | |
| if has_ve(i, config.n_layer) | |
| }) | |
| self.rotary_seq_len = config.sequence_len * 10 | |
| inv = 1.0 / ( | |
| 10000 ** ( | |
| torch.arange( | |
| 0, | |
| hd, | |
| 2, | |
| dtype=torch.float32, | |
| ) / hd | |
| ) | |
| ) | |
| freqs = torch.outer( | |
| torch.arange( | |
| self.rotary_seq_len, | |
| dtype=torch.float32, | |
| ), | |
| inv, | |
| ) | |
| self.register_buffer( | |
| "cos", | |
| freqs.cos()[None, :, None, :], | |
| persistent=False, | |
| ) | |
| self.register_buffer( | |
| "sin", | |
| freqs.sin()[None, :, None, :], | |
| persistent=False, | |
| ) | |
| def forward( | |
| self, | |
| idx, | |
| kv_cache=None, | |
| position_id=0, | |
| ): | |
| B, T = idx.size() | |
| cs = ( | |
| self.cos[:, position_id:position_id + T], | |
| self.sin[:, position_id:position_id + T], | |
| ) | |
| x = norm( | |
| self.transformer.wte(idx) | |
| ) | |
| x0 = x | |
| for i, b in enumerate(self.transformer.h): | |
| x = ( | |
| self.resid_lambdas[i] * x | |
| + self.x0_lambdas[i] * x0 | |
| ) | |
| ve = ( | |
| self.value_embeds[str(i)](idx) | |
| if str(i) in self.value_embeds | |
| else None | |
| ) | |
| layer_cache = ( | |
| kv_cache[i] | |
| if kv_cache is not None | |
| else None | |
| ) | |
| x = b( | |
| x, | |
| ve, | |
| cs, | |
| self.window_sizes[i], | |
| layer_cache, | |
| position_id, | |
| ) | |
| logits = self.lm_head(norm(x)) | |
| logits = logits[..., :self.config.vocab_size] | |
| return 15 * torch.tanh(logits / 15) | |
| def generate( | |
| self, | |
| tokens, | |
| max_tokens=512, | |
| temperature=0.7, | |
| top_k=50, | |
| ): | |
| ids = torch.tensor( | |
| [tokens], | |
| dtype=torch.long, | |
| device=device, | |
| ) | |
| kv_cache = [ | |
| {} | |
| for _ in range(self.config.n_layer) | |
| ] | |
| position_id = 0 | |
| for _ in range(max_tokens): | |
| curr_ids = ( | |
| ids | |
| if position_id == 0 | |
| else ids[:, -1:] | |
| ) | |
| logits = self.forward( | |
| curr_ids, | |
| kv_cache=kv_cache, | |
| position_id=position_id, | |
| )[:, -1, :] | |
| if top_k > 0: | |
| v, _ = torch.topk( | |
| logits, | |
| min(top_k, logits.size(-1)) | |
| ) | |
| logits[ | |
| logits < v[:, [-1]] | |
| ] = -float("Inf") | |
| if temperature > 0: | |
| nid = torch.multinomial( | |
| F.softmax( | |
| logits / temperature, | |
| dim=-1, | |
| ), | |
| 1, | |
| ) | |
| else: | |
| nid = torch.argmax( | |
| logits, | |
| dim=-1, | |
| keepdim=True, | |
| ) | |
| ids = torch.cat( | |
| (ids, nid), | |
| dim=1, | |
| ) | |
| position_id += curr_ids.size(1) | |
| yield nid.item() | |
| # ============================================================================ | |
| # LOAD MODEL | |
| # ============================================================================ | |
| meta = json.load(open(meta_path)) | |
| mc = meta["model_config"] | |
| config = GPTConfig(**{ | |
| k: mc[k] | |
| for k in [ | |
| "sequence_len", | |
| "vocab_size", | |
| "n_layer", | |
| "n_head", | |
| "n_kv_head", | |
| "n_embd", | |
| ] | |
| }) | |
| config.window_pattern = mc.get( | |
| "window_pattern", | |
| "L", | |
| ) | |
| model = GPT(config) | |
| print("Loading weights...") | |
| raw_state_dict = torch.load( | |
| weights_path, | |
| map_location="cpu", | |
| weights_only=True, | |
| ) | |
| model_state = model.state_dict() | |
| for key in list(model_state.keys()): | |
| if key not in raw_state_dict: | |
| continue | |
| tensor = raw_state_dict[key] | |
| if tensor.dtype == torch.int8: | |
| scale_key = key + "_scale" | |
| if scale_key in raw_state_dict: | |
| tensor = ( | |
| tensor.float() | |
| * raw_state_dict[scale_key].float() | |
| ) | |
| model_state[key].copy_( | |
| tensor.to(model_state[key].dtype) | |
| ) | |
| print("Moving model to device...") | |
| model = model.to( | |
| device=device, | |
| dtype=dtype, | |
| ) | |
| model.eval() | |
| del raw_state_dict | |
| gc.collect() | |
| if device == "cuda": | |
| torch.cuda.empty_cache() | |
| # ============================================================================ | |
| # TOKENIZER | |
| # ============================================================================ | |
| with open(tokenizer_path, "rb") as f: | |
| enc = pickle.load(f) | |
| BOS = enc.encode_single_token("<|bos|>") | |
| USER_START = enc.encode_single_token("<|user_start|>") | |
| USER_END = enc.encode_single_token("<|user_end|>") | |
| ASST_START = enc.encode_single_token("<|assistant_start|>") | |
| ASST_END = enc.encode_single_token("<|assistant_end|>") | |
| # ============================================================================ | |
| # CHAT | |
| # ============================================================================ | |
| def respond(message, history): | |
| if not message.strip(): | |
| yield "" | |
| return | |
| ids = [BOS] | |
| ids += [USER_START] | |
| ids += enc.encode_ordinary(message) | |
| ids += [USER_END] | |
| ids += [ASST_START] | |
| generated = [] | |
| for token in model.generate( | |
| ids, | |
| max_tokens=2048, | |
| temperature=0.5, | |
| top_k=50, | |
| ): | |
| if token == ASST_END: | |
| break | |
| generated.append(token) | |
| partial = enc.decode(generated) | |
| yield partial | |
| # ============================================================================ | |
| # UI | |
| # ============================================================================ | |
| css = """ | |
| body { | |
| background: #0f172a; | |
| } | |
| .gradio-container { | |
| max-width: 1100px !important; | |
| margin: auto; | |
| } | |
| footer { | |
| display: none !important; | |
| } | |
| .message.user { | |
| background: #2563eb !important; | |
| color: white !important; | |
| border-radius: 18px !important; | |
| } | |
| .message.bot { | |
| background: #1e293b !important; | |
| color: #f8fafc !important; | |
| border-radius: 18px !important; | |
| } | |
| textarea { | |
| background: #111827 !important; | |
| color: white !important; | |
| border-radius: 16px !important; | |
| border: 1px solid #334155 !important; | |
| } | |
| """ | |
| theme = gr.themes.Soft( | |
| primary_hue="blue", | |
| secondary_hue="slate", | |
| neutral_hue="slate", | |
| ) | |
| demo = gr.ChatInterface( | |
| fn=respond, | |
| title="Treevik-2 (2.5B)", | |
| description=""" | |
| <div style='text-align:center'> | |
| <h3>By PlusB Intelligence Lab</h3> | |
| <p>Instruction Tuned 2.5B Parameter Model</p> | |
| </div> | |
| """, | |
| chatbot=gr.Chatbot( | |
| height=650, | |
| show_copy_button=True, | |
| bubble_full_width=False, | |
| type="messages", | |
| ), | |
| textbox=gr.Textbox( | |
| placeholder="Talk to Treevik...", | |
| container=False, | |
| ), | |
| theme=theme, | |
| css=css, | |
| ) | |
| demo.queue() | |
| demo.launch( | |
| server_name="0.0.0.0", | |
| server_port=7860, | |
| ) | |