Commit ·
4d7ebea
1
Parent(s): e26eba6
Load adapters on CPU before moving to cuda (ZeroGPU fix)
Browse files
app.py
CHANGED
|
@@ -13,38 +13,29 @@ BASE_7B = "Qwen/Qwen2.5-Coder-7B-Instruct"
|
|
| 13 |
BASE_15B = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
|
| 14 |
|
| 15 |
tok7 = AutoTokenizer.from_pretrained(BASE_7B)
|
| 16 |
-
m7 = AutoModelForCausalLM.from_pretrained(BASE_7B,
|
| 17 |
m7 = PeftModel.from_pretrained(m7, "decoded-cipher/nodrix-coder-7b-lora-v2", adapter_name="v2")
|
| 18 |
m7.load_adapter("decoded-cipher/nodrix-coder-7b-lora-v3", adapter_name="v3")
|
|
|
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
global _m15
|
| 25 |
-
if _m15 is None:
|
| 26 |
-
tok = AutoTokenizer.from_pretrained(BASE_15B)
|
| 27 |
-
model = AutoModelForCausalLM.from_pretrained(BASE_15B, torch_dtype=torch.bfloat16).to("cuda")
|
| 28 |
-
model = PeftModel.from_pretrained(model, "decoded-cipher/nodrix-coder-1.5b-lora-v1", adapter_name="v1")
|
| 29 |
-
_m15 = (model, tok)
|
| 30 |
-
return _m15
|
| 31 |
-
|
| 32 |
|
| 33 |
MODELS = {
|
| 34 |
-
"Base 7B (no fine-tune)": (
|
| 35 |
-
"v2 · 7B": (
|
| 36 |
-
"v3 · 7B ★": (
|
| 37 |
-
"Base 1.5B (no fine-tune)": (
|
| 38 |
-
"v1 · 1.5B": (
|
| 39 |
}
|
| 40 |
CHOICES = list(MODELS)
|
| 41 |
|
| 42 |
|
| 43 |
@spaces.GPU(duration=120)
|
| 44 |
def run(choice, question):
|
| 45 |
-
|
| 46 |
-
model, tok = (m7, tok7) if which == "7b" else base15()
|
| 47 |
-
|
| 48 |
ids = tok.apply_chat_template(
|
| 49 |
[{"role": "system", "content": SYSTEM}, {"role": "user", "content": question}],
|
| 50 |
add_generation_prompt=True, return_tensors="pt",
|
|
|
|
| 13 |
BASE_15B = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
|
| 14 |
|
| 15 |
tok7 = AutoTokenizer.from_pretrained(BASE_7B)
|
| 16 |
+
m7 = AutoModelForCausalLM.from_pretrained(BASE_7B, dtype=torch.bfloat16)
|
| 17 |
m7 = PeftModel.from_pretrained(m7, "decoded-cipher/nodrix-coder-7b-lora-v2", adapter_name="v2")
|
| 18 |
m7.load_adapter("decoded-cipher/nodrix-coder-7b-lora-v3", adapter_name="v3")
|
| 19 |
+
m7 = m7.to("cuda")
|
| 20 |
|
| 21 |
+
tok15 = AutoTokenizer.from_pretrained(BASE_15B)
|
| 22 |
+
m15 = AutoModelForCausalLM.from_pretrained(BASE_15B, dtype=torch.bfloat16)
|
| 23 |
+
m15 = PeftModel.from_pretrained(m15, "decoded-cipher/nodrix-coder-1.5b-lora-v1", adapter_name="v1")
|
| 24 |
+
m15 = m15.to("cuda")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
MODELS = {
|
| 27 |
+
"Base 7B (no fine-tune)": (m7, tok7, None),
|
| 28 |
+
"v2 · 7B": (m7, tok7, "v2"),
|
| 29 |
+
"v3 · 7B ★": (m7, tok7, "v3"),
|
| 30 |
+
"Base 1.5B (no fine-tune)": (m15, tok15, None),
|
| 31 |
+
"v1 · 1.5B": (m15, tok15, "v1"),
|
| 32 |
}
|
| 33 |
CHOICES = list(MODELS)
|
| 34 |
|
| 35 |
|
| 36 |
@spaces.GPU(duration=120)
|
| 37 |
def run(choice, question):
|
| 38 |
+
model, tok, adapter = MODELS[choice]
|
|
|
|
|
|
|
| 39 |
ids = tok.apply_chat_template(
|
| 40 |
[{"role": "system", "content": SYSTEM}, {"role": "user", "content": question}],
|
| 41 |
add_generation_prompt=True, return_tensors="pt",
|