eleeter commited on
Commit
1894710
·
verified ·
1 Parent(s): 1c2e69f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +8 -40
app.py CHANGED
@@ -3,53 +3,21 @@ from llama_cpp import Llama
3
  from huggingface_hub import hf_hub_download
4
 
5
  print("Loading model... please wait.")
 
6
  model_path = hf_hub_download(
7
  repo_id="mradermacher/Qwen2.5-Coder-32B-Instruct-GGUF",
8
- filename="Qwen2.5-Coder-32B-Instruct.IQ2_M.gguf"
9
- )
10
-
11
- llm = Llama(
12
- model_path=model_path,
13
- n_ctx=2048,
14
- n_threads=2
15
  )
16
 
17
- def respond(message, history, system_message, max_tokens, temperature, top_p):
18
- formatted_messages = [{"role": "system", "content": system_message}]
19
- for user_msg, assistant_msg in history:
20
- formatted_messages.append({"role": "user", "content": user_msg})
21
- formatted_messages.append({"role": "assistant", "content": assistant_msg})
22
- formatted_messages.append({"role": "user", "content": message})
23
 
24
- response_text = ""
 
25
 
26
- stream = llm.create_chat_completion(
27
- messages=formatted_messages,
28
- max_tokens=max_tokens,
29
- temperature=temperature,
30
- top_p=top_p,
31
- stream=True
32
- )
33
 
34
- for chunk in stream:
35
- if "content" in chunk["choices"][0]["delta"]:
36
- token = chunk["choices"][0]["delta"]["content"]
37
- response_text += token
38
- yield response_text
39
-
40
- # --- Gradio UI Setup ---
41
- with gr.Blocks() as demo:
42
- gr.Markdown("Qwen2.5-Coder-32B (2-bit)")
43
-
44
- chatbot = gr.ChatInterface(
45
- respond,
46
- additional_inputs=[
47
- gr.Textbox(value="You are an Elite Minecraft Modder.", label="System message"),
48
- gr.Slider(minimum=1, maximum=1024, value=512, step=1, label="Max new tokens"),
49
- gr.Slider(minimum=0.1, maximum=1.0, value=0.7, step=0.1, label="Temperature"),
50
- gr.Slider(minimum=0.1, maximum=1.0, value=0.9, step=0.05, label="Top-p"),
51
- ],
52
- )
53
 
54
  if __name__ == "__main__":
55
  demo.launch()
 
3
  from huggingface_hub import hf_hub_download
4
 
5
  print("Loading model... please wait.")
6
+
7
  model_path = hf_hub_download(
8
  repo_id="mradermacher/Qwen2.5-Coder-32B-Instruct-GGUF",
9
+ filename="Qwen2.5-Coder-32B-Instruct.Q2_K.gguf"
 
 
 
 
 
 
10
  )
11
 
12
+ llm = Llama(model_path=model_path, n_ctx=2048)
 
 
 
 
 
13
 
14
+ def respond(message, history):
15
+ prompt = f"System: You are a helpful AI.\nUser: {message}\nAssistant:"
16
 
17
+ output = llm(prompt, max_tokens=512, stop=["User:"], echo=False)
18
+ return output["choices"][0]["text"]
 
 
 
 
 
19
 
20
+ demo = gr.ChatInterface(fn=respond, title="Qwen 32B Coder (CPU)")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
 
22
  if __name__ == "__main__":
23
  demo.launch()