""" Treevik-2 2.5B Instruct HF Spaces Ready GPU + CPU Auto Detect Modern ChatGPT Style UI """ import gc import json import pickle from dataclasses import dataclass import gradio as gr import spaces import torch import torch.nn as nn import torch.nn.functional as F from huggingface_hub import hf_hub_download # ============================================================================ # DEVICE # ============================================================================ torch.set_grad_enabled(False) if torch.cuda.is_available(): device = "cuda" dtype = torch.float16 torch.backends.cuda.matmul.allow_tf32 = True else: device = "cpu" dtype = torch.float32 print(f"Using device: {device}") print(f"Using dtype: {dtype}") # ============================================================================ # DOWNLOAD MODEL # ============================================================================ REPO_ID = "PlusBIntelligenceLab/Treevik2-2.5B-Instruct" print(f"Downloading model from {REPO_ID}...") meta_path = hf_hub_download(REPO_ID, "meta_sft.json") weights_path = hf_hub_download(REPO_ID, "treevik2-2.5B-Instruct-Colab.pt") tokenizer_path = hf_hub_download(REPO_ID, "tokenizer.pkl") # ============================================================================ # MODEL ARCHITECTURE # ============================================================================ def norm(x): return F.rms_norm(x, (x.size(-1),)) def has_ve(li, nl): return li % 2 == (nl - 1) % 2 def apply_rotary_emb(x, cos, sin): d = x.shape[3] // 2 x1, x2 = x[..., :d], x[..., d:] return torch.cat([x1 * cos + x2 * sin, x1 * (-sin) + x2 * cos], 3) class CausalSelfAttention(nn.Module): def __init__(self, config, layer_idx): super().__init__() self.n_head = config.n_head self.n_kv_head = config.n_kv_head self.n_embd = config.n_embd self.head_dim = self.n_embd // self.n_head self.c_q = nn.Linear( self.n_embd, self.n_head * self.head_dim, bias=False, ) self.c_k = nn.Linear( self.n_embd, self.n_kv_head * self.head_dim, bias=False, ) self.c_v = nn.Linear( self.n_embd, self.n_kv_head * self.head_dim, bias=False, ) self.c_proj = nn.Linear( self.n_embd, self.n_embd, bias=False, ) self.ve_gate = ( nn.Linear(32, self.n_kv_head, bias=False) if has_ve(layer_idx, config.n_layer) else None ) def forward( self, x, ve, cos_sin, window_size, kv_cache=None, position_id=0, ): B, T, C = x.size() q = self.c_q(x).view( B, T, self.n_head, self.head_dim ) k = self.c_k(x).view( B, T, self.n_kv_head, self.head_dim ) v = self.c_v(x).view( B, T, self.n_kv_head, self.head_dim ) if ve is not None: ve = ve.view( B, T, self.n_kv_head, self.head_dim, ) v = v + ( 2 * torch.sigmoid( self.ve_gate(x[..., :32]) ) ).unsqueeze(-1) * ve cos, sin = cos_sin q = apply_rotary_emb(q, cos, sin) k = apply_rotary_emb(k, cos, sin) q = norm(q) k = norm(k) if kv_cache is not None: if position_id == 0: kv_cache["k"] = k kv_cache["v"] = v else: kv_cache["k"] = torch.cat( [kv_cache["k"], k], dim=1, ) kv_cache["v"] = torch.cat( [kv_cache["v"], v], dim=1, ) k = kv_cache["k"] v = kv_cache["v"] nr = self.n_head // self.n_kv_head if nr > 1: k = ( k.unsqueeze(3) .expand( B, k.size(1), self.n_kv_head, nr, self.head_dim, ) .reshape( B, k.size(1), self.n_head, self.head_dim, ) ) v = ( v.unsqueeze(3) .expand( B, v.size(1), self.n_kv_head, nr, self.head_dim, ) .reshape( B, v.size(1), self.n_head, self.head_dim, ) ) q = q.transpose(1, 2) k = k.transpose(1, 2) v = v.transpose(1, 2) y = F.scaled_dot_product_attention( q, k, v, is_causal=(T > 1), ) y = ( y.transpose(1, 2) .contiguous() .view(B, T, -1) ) return self.c_proj(y) class MLP(nn.Module): def __init__(self, config): super().__init__() self.c_fc = nn.Linear( config.n_embd, 4 * config.n_embd, bias=False, ) self.c_proj = nn.Linear( 4 * config.n_embd, config.n_embd, bias=False, ) def forward(self, x): return self.c_proj( F.relu(self.c_fc(x)).square() ) class Block(nn.Module): def __init__(self, config, li): super().__init__() self.attn = CausalSelfAttention(config, li) self.mlp = MLP(config) def forward( self, x, ve, cs, ws, kv_cache=None, position_id=0, ): x = x + self.attn( norm(x), ve, cs, ws, kv_cache, position_id, ) x = x + self.mlp(norm(x)) return x @dataclass class GPTConfig: sequence_len: int = 2048 vocab_size: int = 65536 n_layer: int = 26 n_head: int = 13 n_kv_head: int = 13 n_embd: int = 1664 window_pattern: str = "L" class GPT(nn.Module): def __init__(self, config): super().__init__() self.config = config pat = config.window_pattern.upper() lw = config.sequence_len sw = config.sequence_len // 2 self.window_sizes = [ ( (sw, 0) if c == "S" else (lw, 0) ) for c in (pat * config.n_layer)[:config.n_layer] ] self.window_sizes[-1] = (lw, 0) pad = ((config.vocab_size + 63) // 64) * 64 self.transformer = nn.ModuleDict({ "wte": nn.Embedding( pad, config.n_embd, ), "h": nn.ModuleList([ Block(config, i) for i in range(config.n_layer) ]), }) self.lm_head = nn.Linear( config.n_embd, pad, bias=False, ) self.resid_lambdas = nn.Parameter( torch.ones(config.n_layer) ) self.x0_lambdas = nn.Parameter( torch.zeros(config.n_layer) ) hd = config.n_embd // config.n_head self.value_embeds = nn.ModuleDict({ str(i): nn.Embedding( pad, config.n_kv_head * hd, ) for i in range(config.n_layer) if has_ve(i, config.n_layer) }) self.rotary_seq_len = config.sequence_len * 10 inv = 1.0 / ( 10000 ** ( torch.arange( 0, hd, 2, dtype=torch.float32, ) / hd ) ) freqs = torch.outer( torch.arange( self.rotary_seq_len, dtype=torch.float32, ), inv, ) self.register_buffer( "cos", freqs.cos()[None, :, None, :], persistent=False, ) self.register_buffer( "sin", freqs.sin()[None, :, None, :], persistent=False, ) def forward( self, idx, kv_cache=None, position_id=0, ): B, T = idx.size() cs = ( self.cos[:, position_id:position_id + T], self.sin[:, position_id:position_id + T], ) x = norm( self.transformer.wte(idx) ) x0 = x for i, b in enumerate(self.transformer.h): x = ( self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 ) ve = ( self.value_embeds[str(i)](idx) if str(i) in self.value_embeds else None ) layer_cache = ( kv_cache[i] if kv_cache is not None else None ) x = b( x, ve, cs, self.window_sizes[i], layer_cache, position_id, ) logits = self.lm_head(norm(x)) logits = logits[..., :self.config.vocab_size] return 15 * torch.tanh(logits / 15) @torch.inference_mode() def generate( self, tokens, max_tokens=512, temperature=0.7, top_k=50, ): ids = torch.tensor( [tokens], dtype=torch.long, device=device, ) kv_cache = [ {} for _ in range(self.config.n_layer) ] position_id = 0 for _ in range(max_tokens): curr_ids = ( ids if position_id == 0 else ids[:, -1:] ) logits = self.forward( curr_ids, kv_cache=kv_cache, position_id=position_id, )[:, -1, :] if top_k > 0: v, _ = torch.topk( logits, min(top_k, logits.size(-1)) ) logits[ logits < v[:, [-1]] ] = -float("Inf") if temperature > 0: nid = torch.multinomial( F.softmax( logits / temperature, dim=-1, ), 1, ) else: nid = torch.argmax( logits, dim=-1, keepdim=True, ) ids = torch.cat( (ids, nid), dim=1, ) position_id += curr_ids.size(1) yield nid.item() # ============================================================================ # LOAD MODEL # ============================================================================ meta = json.load(open(meta_path)) mc = meta["model_config"] config = GPTConfig(**{ k: mc[k] for k in [ "sequence_len", "vocab_size", "n_layer", "n_head", "n_kv_head", "n_embd", ] }) config.window_pattern = mc.get( "window_pattern", "L", ) model = GPT(config) print("Loading weights...") raw_state_dict = torch.load( weights_path, map_location="cpu", weights_only=True, ) model_state = model.state_dict() for key in list(model_state.keys()): if key not in raw_state_dict: continue tensor = raw_state_dict[key] if tensor.dtype == torch.int8: scale_key = key + "_scale" if scale_key in raw_state_dict: tensor = ( tensor.float() * raw_state_dict[scale_key].float() ) model_state[key].copy_( tensor.to(model_state[key].dtype) ) print("Moving model to device...") model = model.to( device=device, dtype=dtype, ) model.eval() del raw_state_dict gc.collect() if device == "cuda": torch.cuda.empty_cache() # ============================================================================ # TOKENIZER # ============================================================================ with open(tokenizer_path, "rb") as f: enc = pickle.load(f) BOS = enc.encode_single_token("<|bos|>") USER_START = enc.encode_single_token("<|user_start|>") USER_END = enc.encode_single_token("<|user_end|>") ASST_START = enc.encode_single_token("<|assistant_start|>") ASST_END = enc.encode_single_token("<|assistant_end|>") # ============================================================================ # CHAT # ============================================================================ @spaces.GPU def respond(message, history): if not message.strip(): yield "" return ids = [BOS] ids += [USER_START] ids += enc.encode_ordinary(message) ids += [USER_END] ids += [ASST_START] generated = [] for token in model.generate( ids, max_tokens=2048, temperature=0.5, top_k=50, ): if token == ASST_END: break generated.append(token) partial = enc.decode(generated) yield partial # ============================================================================ # UI # ============================================================================ css = """ body { background: #0f172a; } .gradio-container { max-width: 1100px !important; margin: auto; } footer { display: none !important; } .message.user { background: #2563eb !important; color: white !important; border-radius: 18px !important; } .message.bot { background: #1e293b !important; color: #f8fafc !important; border-radius: 18px !important; } textarea { background: #111827 !important; color: white !important; border-radius: 16px !important; border: 1px solid #334155 !important; } """ theme = gr.themes.Soft( primary_hue="blue", secondary_hue="slate", neutral_hue="slate", ) demo = gr.ChatInterface( fn=respond, title="Treevik-2 (2.5B)", description="""
Instruction Tuned 2.5B Parameter Model