eleeter commited on
Commit
852f6c1
·
verified ·
1 Parent(s): 1894710

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +27 -10
app.py CHANGED
@@ -2,22 +2,39 @@ import gradio as gr
2
  from llama_cpp import Llama
3
  from huggingface_hub import hf_hub_download
4
 
5
- print("Loading model... please wait.")
6
-
7
  model_path = hf_hub_download(
8
- repo_id="mradermacher/Qwen2.5-Coder-32B-Instruct-GGUF",
9
- filename="Qwen2.5-Coder-32B-Instruct.Q2_K.gguf"
10
  )
11
 
12
- llm = Llama(model_path=model_path, n_ctx=2048)
 
 
 
 
 
13
 
14
  def respond(message, history):
15
- prompt = f"System: You are a helpful AI.\nUser: {message}\nAssistant:"
16
-
17
- output = llm(prompt, max_tokens=512, stop=["User:"], echo=False)
18
- return output["choices"][0]["text"]
 
 
 
 
 
 
 
 
 
 
 
 
 
19
 
20
- demo = gr.ChatInterface(fn=respond, title="Qwen 32B Coder (CPU)")
21
 
22
  if __name__ == "__main__":
23
  demo.launch()
 
2
  from llama_cpp import Llama
3
  from huggingface_hub import hf_hub_download
4
 
5
+ print("Loading 7B Model into CPU RAM...")
 
6
  model_path = hf_hub_download(
7
+ repo_id="Qwen/Qwen2.5-Coder-7B-Instruct-GGUF",
8
+ filename="qwen2.5-coder-7b-instruct-q4_k_m.gguf"
9
  )
10
 
11
+ llm = Llama(
12
+ model_path=model_path,
13
+ n_ctx=1024,
14
+ n_threads=2,
15
+ n_batch=512
16
+ )
17
 
18
  def respond(message, history):
19
+ prompt = f"<|im_start|>system\nYou are a professional coder.<|im_end|>\n"
20
+ for user_msg, assistant_msg in history:
21
+ prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n<|im_start|>assistant\n{assistant_msg}<|im_end|>\n"
22
+ prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
23
+
24
+ response_text = ""
25
+ stream = llm(
26
+ prompt,
27
+ max_tokens=512,
28
+ stop=["<|im_end|>"],
29
+ stream=True
30
+ )
31
+
32
+ for chunk in stream:
33
+ token = chunk["choices"][0]["text"]
34
+ response_text += token
35
+ yield response_text
36
 
37
+ demo = gr.ChatInterface(respond)
38
 
39
  if __name__ == "__main__":
40
  demo.launch()