Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,8 +1,6 @@
|
|
| 1 |
# ================================================================
|
| 2 |
-
# CodeSymb AI โ
|
| 3 |
-
#
|
| 4 |
-
# โ
Loads 141MB Chat LoRA
|
| 5 |
-
# โ
Merges them mathematically for Native Speed!
|
| 6 |
# NiftyOracleAI ๐ฎ๐ณ | codesymb.com ๐
|
| 7 |
# ================================================================
|
| 8 |
|
|
@@ -18,13 +16,13 @@ from huggingface_hub import hf_hub_download, login
|
|
| 18 |
HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
| 19 |
USERNAME = "NiftyOracleAI"
|
| 20 |
REPO_ID = f"{USERNAME}/NiftyOracle-Coder-30B"
|
| 21 |
-
LORA_REPO = f"{USERNAME}/codesymb-
|
| 22 |
TOK_REPO = f"{USERNAME}/niftyoracle-tokenizer"
|
| 23 |
|
| 24 |
if HF_TOKEN: login(token=HF_TOKEN)
|
| 25 |
|
| 26 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 27 |
-
# MODEL ARCHITECTURE
|
| 28 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 29 |
class RMSNorm(nn.Module):
|
| 30 |
def __init__(self, d, eps=1e-6):
|
|
@@ -39,8 +37,8 @@ class RoPE(nn.Module):
|
|
| 39 |
super().__init__()
|
| 40 |
inv = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
|
| 41 |
emb = torch.cat([torch.outer(torch.arange(max_len).float(), inv)] * 2, dim=-1)
|
| 42 |
-
self.register_buffer("cos", emb.cos()[None, None].
|
| 43 |
-
self.register_buffer("sin", emb.sin()[None, None].
|
| 44 |
def forward(self, n): return self.cos[:, :, :n], self.sin[:, :, :n]
|
| 45 |
|
| 46 |
def rot(x):
|
|
@@ -98,7 +96,7 @@ class CodeSymb1B(nn.Module):
|
|
| 98 |
return self.head(self.norm(x))
|
| 99 |
|
| 100 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 101 |
-
# LOAD BASE
|
| 102 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 103 |
print("โณ Loading Tokenizer...")
|
| 104 |
tok_path = hf_hub_download(TOK_REPO, "niftyoracle_tokenizer.model", token=HF_TOKEN or None)
|
|
@@ -108,61 +106,87 @@ print("โณ Loading Base Model (240k steps)...")
|
|
| 108 |
model = CodeSymb1B(vocab=sp.get_piece_size())
|
| 109 |
base_ckpt = hf_hub_download(REPO_ID, "checkpoints/latest.pt", token=HF_TOKEN or None)
|
| 110 |
state = torch.load(base_ckpt, map_location="cpu", weights_only=False)
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
try:
|
| 116 |
-
# Use safetensors from the PEFT upload!
|
| 117 |
-
from safetensors.torch import load_file
|
| 118 |
-
lora_ckpt = hf_hub_download(LORA_REPO, "chat-behavior-lora/adapter_model.safetensors", token=HF_TOKEN or None)
|
| 119 |
-
lora_state = load_file(lora_ckpt)
|
| 120 |
-
|
| 121 |
-
# Mathematically fuse the LoRA matrices (W = W + A*B * scale)
|
| 122 |
-
scale = 200 / 100 # alpha / rank
|
| 123 |
-
with torch.no_grad():
|
| 124 |
-
for i, block in enumerate(model.blocks):
|
| 125 |
-
if f"base_model.model.blocks.{i}.att.q.lora_A.weight" in lora_state:
|
| 126 |
-
block.att.q.weight += (lora_state[f"base_model.model.blocks.{i}.att.q.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.att.q.lora_A.weight"]) * scale
|
| 127 |
-
block.att.k.weight += (lora_state[f"base_model.model.blocks.{i}.att.k.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.att.k.lora_A.weight"]) * scale
|
| 128 |
-
block.att.v.weight += (lora_state[f"base_model.model.blocks.{i}.att.v.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.att.v.lora_A.weight"]) * scale
|
| 129 |
-
block.att.o.weight += (lora_state[f"base_model.model.blocks.{i}.att.o.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.att.o.lora_A.weight"]) * scale
|
| 130 |
-
block.ff.g.weight += (lora_state[f"base_model.model.blocks.{i}.ff.g.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.ff.g.lora_A.weight"]) * scale
|
| 131 |
-
block.ff.u.weight += (lora_state[f"base_model.model.blocks.{i}.ff.u.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.ff.u.lora_A.weight"]) * scale
|
| 132 |
-
block.ff.d.weight += (lora_state[f"base_model.model.blocks.{i}.ff.d.lora_B.weight"] @ lora_state[f"base_model.model.blocks.{i}.ff.d.lora_A.weight"]) * scale
|
| 133 |
-
print("โ
Chat LoRA Fused Successfully!")
|
| 134 |
-
except Exception as e:
|
| 135 |
-
print(f"โ ๏ธ Could not load LoRA (Make sure path is correct!): {e}")
|
| 136 |
|
| 137 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 138 |
-
model = model.to(device).
|
| 139 |
-
|
|
|
|
| 140 |
|
| 141 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 142 |
-
#
|
| 143 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
@spaces.GPU(duration=60)
|
| 151 |
-
def chat_with_codesymb(message: str, history: list) -> str:
|
| 152 |
-
|
| 153 |
-
|
| 154 |
|
| 155 |
-
|
| 156 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
if len(prompt_ids) > 1024: prompt_ids = prompt_ids[-1024:]
|
| 158 |
|
| 159 |
inp = torch.tensor([prompt_ids], dtype=torch.long).to(device)
|
| 160 |
-
END_SEQ = sp.encode("<|endoftext|>")
|
| 161 |
generated = []
|
| 162 |
-
|
| 163 |
with torch.no_grad():
|
| 164 |
for _ in range(512):
|
| 165 |
-
with torch.autocast("cuda"
|
| 166 |
logits = model(inp)[0, -1, :].float()
|
| 167 |
|
| 168 |
for gid in set(generated[-50:]):
|
|
@@ -176,8 +200,7 @@ def chat_with_codesymb(message: str, history: list) -> str:
|
|
| 176 |
sorted_logits, sorted_idx = torch.sort(logits, descending=True)
|
| 177 |
cum_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
| 178 |
remove = cum_probs > 0.90
|
| 179 |
-
remove[
|
| 180 |
-
remove[..., 0] = 0
|
| 181 |
sorted_logits[remove] = -float("inf")
|
| 182 |
|
| 183 |
logits_filtered = torch.zeros_like(logits)
|
|
@@ -185,24 +208,54 @@ def chat_with_codesymb(message: str, history: list) -> str:
|
|
| 185 |
probs = F.softmax(logits_filtered, dim=-1)
|
| 186 |
|
| 187 |
next_t = torch.multinomial(probs, 1).item()
|
|
|
|
| 188 |
if len(generated) >= 3 and generated[-1] == next_t and generated[-2] == next_t:
|
| 189 |
break
|
| 190 |
|
| 191 |
generated.append(next_t)
|
| 192 |
inp = torch.cat([inp, torch.tensor([[next_t]], device=device)], dim=1)
|
| 193 |
|
| 194 |
-
|
|
|
|
|
|
|
| 195 |
|
| 196 |
response = sp.decode(generated).strip()
|
| 197 |
for tag in ["<|user|>", "<|system|>", "<|assistant|>", "<|endoftext|>"]:
|
| 198 |
-
response = response.split(tag)[0]
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
|
| 207 |
if __name__ == "__main__":
|
| 208 |
demo.launch()
|
|
|
|
| 1 |
# ================================================================
|
| 2 |
+
# CodeSymb AI โ THE DYNAMIC LORA SHAPE-SHIFTER ๐ฆ๐ฅ
|
| 3 |
+
# Runs natively on Hugging Face Spaces!
|
|
|
|
|
|
|
| 4 |
# NiftyOracleAI ๐ฎ๐ณ | codesymb.com ๐
|
| 5 |
# ================================================================
|
| 6 |
|
|
|
|
| 16 |
HF_TOKEN = os.environ.get("HF_TOKEN", "")
|
| 17 |
USERNAME = "NiftyOracleAI"
|
| 18 |
REPO_ID = f"{USERNAME}/NiftyOracle-Coder-30B"
|
| 19 |
+
LORA_REPO = f"{USERNAME}/codesymb-lora-vault"
|
| 20 |
TOK_REPO = f"{USERNAME}/niftyoracle-tokenizer"
|
| 21 |
|
| 22 |
if HF_TOKEN: login(token=HF_TOKEN)
|
| 23 |
|
| 24 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 25 |
+
# MODEL ARCHITECTURE (T4/A100 Float16 Compatible)
|
| 26 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 27 |
class RMSNorm(nn.Module):
|
| 28 |
def __init__(self, d, eps=1e-6):
|
|
|
|
| 37 |
super().__init__()
|
| 38 |
inv = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
|
| 39 |
emb = torch.cat([torch.outer(torch.arange(max_len).float(), inv)] * 2, dim=-1)
|
| 40 |
+
self.register_buffer("cos", emb.cos()[None, None].half())
|
| 41 |
+
self.register_buffer("sin", emb.sin()[None, None].half())
|
| 42 |
def forward(self, n): return self.cos[:, :, :n], self.sin[:, :, :n]
|
| 43 |
|
| 44 |
def rot(x):
|
|
|
|
| 96 |
return self.head(self.norm(x))
|
| 97 |
|
| 98 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 99 |
+
# LOAD BASE MODEL INTO RAM ONCE
|
| 100 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 101 |
print("โณ Loading Tokenizer...")
|
| 102 |
tok_path = hf_hub_download(TOK_REPO, "niftyoracle_tokenizer.model", token=HF_TOKEN or None)
|
|
|
|
| 106 |
model = CodeSymb1B(vocab=sp.get_piece_size())
|
| 107 |
base_ckpt = hf_hub_download(REPO_ID, "checkpoints/latest.pt", token=HF_TOKEN or None)
|
| 108 |
state = torch.load(base_ckpt, map_location="cpu", weights_only=False)
|
| 109 |
+
|
| 110 |
+
# Keep a pristine copy of the base weights in RAM so we can swap LoRAs infinitely!
|
| 111 |
+
BASE_WEIGHTS = {k: v for k, v in state.get("model", state).items() if not k.endswith((".pe.cos", ".pe.sin"))}
|
| 112 |
+
model.load_state_dict(BASE_WEIGHTS, strict=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
|
| 114 |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 115 |
+
model = model.to(device).half().eval()
|
| 116 |
+
CURRENT_LORA = "None"
|
| 117 |
+
print("โ
Base Model is LIVE!")
|
| 118 |
|
| 119 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 120 |
+
# DYNAMIC LORA HOT-SWAPPER
|
| 121 |
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 122 |
+
def switch_lora(expert_name):
|
| 123 |
+
global CURRENT_LORA
|
| 124 |
+
if expert_name == CURRENT_LORA:
|
| 125 |
+
return # Already loaded!
|
| 126 |
+
|
| 127 |
+
print(f"๐ Swapping Brain to: {expert_name}...")
|
| 128 |
+
|
| 129 |
+
# 1. Reset model to pristine Base Weights
|
| 130 |
+
model.load_state_dict(BASE_WEIGHTS, strict=False)
|
| 131 |
+
|
| 132 |
+
if expert_name == "Base Model (No Expert)":
|
| 133 |
+
CURRENT_LORA = expert_name
|
| 134 |
+
return
|
| 135 |
|
| 136 |
+
# 2. Download and fuse the requested LoRA!
|
| 137 |
+
try:
|
| 138 |
+
lora_ckpt = hf_hub_download(LORA_REPO, f"loras/{expert_name}/adapter.pt", token=HF_TOKEN or None)
|
| 139 |
+
lora_state = torch.load(lora_ckpt, map_location="cpu", weights_only=False)
|
| 140 |
+
|
| 141 |
+
scale = 200 / 100 # Alpha / Rank for our H100 LoRAs
|
| 142 |
+
with torch.no_grad():
|
| 143 |
+
for i, block in enumerate(model.blocks):
|
| 144 |
+
for layer_name in ["att.q", "att.k", "att.v", "att.o", "ff.g", "ff.u", "ff.d"]:
|
| 145 |
+
A_key = f"blocks.{i}.{layer_name}.A"
|
| 146 |
+
B_key = f"blocks.{i}.{layer_name}.B"
|
| 147 |
+
|
| 148 |
+
if A_key in lora_state and B_key in lora_state:
|
| 149 |
+
A = lora_state[A_key].float().to(device)
|
| 150 |
+
B = lora_state[B_key].float().to(device)
|
| 151 |
+
delta = (A @ B).T * scale
|
| 152 |
+
|
| 153 |
+
target_layer = getattr(block, layer_name.split(".")[0])
|
| 154 |
+
sub_layer = getattr(target_layer, layer_name.split(".")[1])
|
| 155 |
+
sub_layer.weight.data += delta.to(torch.float16)
|
| 156 |
+
|
| 157 |
+
CURRENT_LORA = expert_name
|
| 158 |
+
print(f"โ
{expert_name} Fused Successfully!")
|
| 159 |
+
except Exception as e:
|
| 160 |
+
print(f"โ ๏ธ Failed to load {expert_name}: {e}")
|
| 161 |
+
|
| 162 |
+
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 163 |
+
# INFERENCE ENGINE
|
| 164 |
+
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 165 |
@spaces.GPU(duration=60)
|
| 166 |
+
def chat_with_codesymb(message: str, history: list, expert_choice: str) -> str:
|
| 167 |
+
# ๐ฅ DYNAMICALLY SHAPE-SHIFT THE BRAIN!
|
| 168 |
+
switch_lora(expert_choice)
|
| 169 |
|
| 170 |
+
system_prompt = (
|
| 171 |
+
f"You are CodeSymb, acting as the {expert_choice}. You are deeply empathetic, "
|
| 172 |
+
"passionate, and highly intelligent. Built by Dev Malkar and Founder Deepakkumar Malkar "
|
| 173 |
+
"at NiftyOracle India. Provide flawless code and do not hallucinate."
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
full_prompt = f"<|system|>\n{system_prompt}\n"
|
| 177 |
+
for user_msg, ai_msg in history[-3:]:
|
| 178 |
+
full_prompt += f"<|user|>\n{user_msg}\n<|assistant|>\n{ai_msg}\n"
|
| 179 |
+
full_prompt += f"<|user|>\n{message}\n<|assistant|>\n"
|
| 180 |
+
|
| 181 |
+
prompt_ids = sp.encode(full_prompt)
|
| 182 |
if len(prompt_ids) > 1024: prompt_ids = prompt_ids[-1024:]
|
| 183 |
|
| 184 |
inp = torch.tensor([prompt_ids], dtype=torch.long).to(device)
|
|
|
|
| 185 |
generated = []
|
| 186 |
+
|
| 187 |
with torch.no_grad():
|
| 188 |
for _ in range(512):
|
| 189 |
+
with torch.autocast("cuda", dtype=torch.float16):
|
| 190 |
logits = model(inp)[0, -1, :].float()
|
| 191 |
|
| 192 |
for gid in set(generated[-50:]):
|
|
|
|
| 200 |
sorted_logits, sorted_idx = torch.sort(logits, descending=True)
|
| 201 |
cum_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
| 202 |
remove = cum_probs > 0.90
|
| 203 |
+
remove[1:] = remove[:-1].clone(); remove[0] = False
|
|
|
|
| 204 |
sorted_logits[remove] = -float("inf")
|
| 205 |
|
| 206 |
logits_filtered = torch.zeros_like(logits)
|
|
|
|
| 208 |
probs = F.softmax(logits_filtered, dim=-1)
|
| 209 |
|
| 210 |
next_t = torch.multinomial(probs, 1).item()
|
| 211 |
+
|
| 212 |
if len(generated) >= 3 and generated[-1] == next_t and generated[-2] == next_t:
|
| 213 |
break
|
| 214 |
|
| 215 |
generated.append(next_t)
|
| 216 |
inp = torch.cat([inp, torch.tensor([[next_t]], device=device)], dim=1)
|
| 217 |
|
| 218 |
+
decoded = sp.decode(generated)
|
| 219 |
+
if "<|endoftext|>" in decoded or "<|user|>" in decoded:
|
| 220 |
+
break
|
| 221 |
|
| 222 |
response = sp.decode(generated).strip()
|
| 223 |
for tag in ["<|user|>", "<|system|>", "<|assistant|>", "<|endoftext|>"]:
|
| 224 |
+
response = response.split(tag)[0].strip()
|
| 225 |
+
|
| 226 |
+
return response
|
| 227 |
+
|
| 228 |
+
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 229 |
+
# GRADIO UI WITH EXPERT DROPDOWN!
|
| 230 |
+
# โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
| 231 |
+
with gr.Blocks(theme=gr.themes.Base(primary_hue="cyan", secondary_hue="blue"), css="body { background: #060D1F !important; color: white !important; }") as demo:
|
| 232 |
+
gr.HTML("""
|
| 233 |
+
<h1 style='text-align:center; color:#00D4FF;'>โก CodeSymb AI: Expert Swarm</h1>
|
| 234 |
+
<p style='text-align:center; color:#A0AEC0;'>
|
| 235 |
+
Built by <strong>Deepakkumar Malkar</strong> at NiftyOracle ๐ฎ๐ณ<br>
|
| 236 |
+
<em>Dynamically swapping Rank-100 LoRAs on the fly!</em>
|
| 237 |
+
</p>
|
| 238 |
+
""")
|
| 239 |
+
|
| 240 |
+
with gr.Row():
|
| 241 |
+
expert_dropdown = gr.Dropdown(
|
| 242 |
+
choices=[
|
| 243 |
+
"Base Model (No Expert)",
|
| 244 |
+
"python_master",
|
| 245 |
+
"javascript_master",
|
| 246 |
+
"cpp_master",
|
| 247 |
+
"react_master",
|
| 248 |
+
"hindi_coding_master"
|
| 249 |
+
],
|
| 250 |
+
value="python_master",
|
| 251 |
+
label="๐ง Select CodeSymb Expert Brain",
|
| 252 |
+
interactive=True
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
gr.ChatInterface(
|
| 256 |
+
fn=chat_with_codesymb,
|
| 257 |
+
additional_inputs=[expert_dropdown]
|
| 258 |
+
)
|
| 259 |
|
| 260 |
if __name__ == "__main__":
|
| 261 |
demo.launch()
|