kovinape commited on
Commit
9891042
·
verified ·
1 Parent(s): 13b331d

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +56 -0
app.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from llama_cpp import Llama
3
+ from huggingface_hub import hf_hub_download
4
+
5
+ model_id = "Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF"
6
+ model_file = "qwen2.5-coder-0.5b-instruct-q4_k_m.gguf"
7
+
8
+ print("Downloading model...")
9
+ model_path = hf_hub_download(repo_id=model_id, filename=model_file)
10
+ print("Loading model...")
11
+ llm = Llama(model_path=model_path, n_ctx=2048, n_threads=2)
12
+
13
+ def generate_code(prompt, max_tokens=512, temperature=0.7, top_p=0.9):
14
+ messages = [
15
+ {"role": "system", "content": "You are a helpful coding assistant."},
16
+ {"role": "user", "content": prompt}
17
+ ]
18
+ output = llm.create_chat_completion(
19
+ messages=messages,
20
+ max_tokens=max_tokens,
21
+ temperature=temperature,
22
+ top_p=top_p
23
+ )
24
+ return output["choices"][0]["message"]["content"]
25
+
26
+ with gr.Blocks(title="Qwen 2.5 Coder 0.5B", theme=gr.themes.Soft()) as demo:
27
+ gr.Markdown("# Qwen 2.5 Coder 0.5B")
28
+ gr.Markdown("Powered by llama.cpp - Running locally on CPU")
29
+
30
+ with gr.Row():
31
+ with gr.Column():
32
+ prompt = gr.Textbox(
33
+ label="Input",
34
+ placeholder="Enter your coding question or prompt...",
35
+ lines=5
36
+ )
37
+ with gr.Row():
38
+ max_tokens = gr.Slider(minimum=64, maximum=2048, value=512, label="Max Tokens")
39
+ temperature = gr.Slider(minimum=0.0, maximum=2.0, value=0.7, label="Temperature")
40
+ top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.9, label="Top P")
41
+ submit = gr.Button("Generate", variant="primary")
42
+ with gr.Column():
43
+ output = gr.Textbox(label="Output", lines=10)
44
+
45
+ submit.click(
46
+ fn=generate_code,
47
+ inputs=[prompt, max_tokens, temperature, top_p],
48
+ outputs=output
49
+ )
50
+ prompt.submit(
51
+ fn=generate_code,
52
+ inputs=[prompt, max_tokens, temperature, top_p],
53
+ outputs=output
54
+ )
55
+
56
+ demo.launch()