decoded-cipher commited on
Commit
4d7ebea
·
1 Parent(s): e26eba6

Load adapters on CPU before moving to cuda (ZeroGPU fix)

Browse files
Files changed (1) hide show
  1. app.py +12 -21
app.py CHANGED
@@ -13,38 +13,29 @@ BASE_7B = "Qwen/Qwen2.5-Coder-7B-Instruct"
13
  BASE_15B = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
14
 
15
  tok7 = AutoTokenizer.from_pretrained(BASE_7B)
16
- m7 = AutoModelForCausalLM.from_pretrained(BASE_7B, torch_dtype=torch.bfloat16).to("cuda")
17
  m7 = PeftModel.from_pretrained(m7, "decoded-cipher/nodrix-coder-7b-lora-v2", adapter_name="v2")
18
  m7.load_adapter("decoded-cipher/nodrix-coder-7b-lora-v3", adapter_name="v3")
 
19
 
20
- _m15 = None
21
-
22
-
23
- def base15():
24
- global _m15
25
- if _m15 is None:
26
- tok = AutoTokenizer.from_pretrained(BASE_15B)
27
- model = AutoModelForCausalLM.from_pretrained(BASE_15B, torch_dtype=torch.bfloat16).to("cuda")
28
- model = PeftModel.from_pretrained(model, "decoded-cipher/nodrix-coder-1.5b-lora-v1", adapter_name="v1")
29
- _m15 = (model, tok)
30
- return _m15
31
-
32
 
33
  MODELS = {
34
- "Base 7B (no fine-tune)": ("7b", None),
35
- "v2 · 7B": ("7b", "v2"),
36
- "v3 · 7B ★": ("7b", "v3"),
37
- "Base 1.5B (no fine-tune)": ("15b", None),
38
- "v1 · 1.5B": ("15b", "v1"),
39
  }
40
  CHOICES = list(MODELS)
41
 
42
 
43
  @spaces.GPU(duration=120)
44
  def run(choice, question):
45
- which, adapter = MODELS[choice]
46
- model, tok = (m7, tok7) if which == "7b" else base15()
47
-
48
  ids = tok.apply_chat_template(
49
  [{"role": "system", "content": SYSTEM}, {"role": "user", "content": question}],
50
  add_generation_prompt=True, return_tensors="pt",
 
13
  BASE_15B = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
14
 
15
  tok7 = AutoTokenizer.from_pretrained(BASE_7B)
16
+ m7 = AutoModelForCausalLM.from_pretrained(BASE_7B, dtype=torch.bfloat16)
17
  m7 = PeftModel.from_pretrained(m7, "decoded-cipher/nodrix-coder-7b-lora-v2", adapter_name="v2")
18
  m7.load_adapter("decoded-cipher/nodrix-coder-7b-lora-v3", adapter_name="v3")
19
+ m7 = m7.to("cuda")
20
 
21
+ tok15 = AutoTokenizer.from_pretrained(BASE_15B)
22
+ m15 = AutoModelForCausalLM.from_pretrained(BASE_15B, dtype=torch.bfloat16)
23
+ m15 = PeftModel.from_pretrained(m15, "decoded-cipher/nodrix-coder-1.5b-lora-v1", adapter_name="v1")
24
+ m15 = m15.to("cuda")
 
 
 
 
 
 
 
 
25
 
26
  MODELS = {
27
+ "Base 7B (no fine-tune)": (m7, tok7, None),
28
+ "v2 · 7B": (m7, tok7, "v2"),
29
+ "v3 · 7B ★": (m7, tok7, "v3"),
30
+ "Base 1.5B (no fine-tune)": (m15, tok15, None),
31
+ "v1 · 1.5B": (m15, tok15, "v1"),
32
  }
33
  CHOICES = list(MODELS)
34
 
35
 
36
  @spaces.GPU(duration=120)
37
  def run(choice, question):
38
+ model, tok, adapter = MODELS[choice]
 
 
39
  ids = tok.apply_chat_template(
40
  [{"role": "system", "content": SYSTEM}, {"role": "user", "content": question}],
41
  add_generation_prompt=True, return_tensors="pt",