eleeter commited on
Commit
eb24ba4
·
verified ·
1 Parent(s): ce23941

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +6 -7
app.py CHANGED
@@ -2,19 +2,18 @@ import gradio as gr
2
  from llama_cpp import Llama
3
  from huggingface_hub import hf_hub_download
4
 
5
- print("Loading 3B Model... ")
 
6
  model_path = hf_hub_download(
7
- repo_id="Qwen/Qwen2.5-Coder-3B-Instruct-GGUF",
8
- filename="qwen2.5-coder-3b-instruct-q5_k_m.gguf"
9
  )
10
 
11
- # Speed Optimizations:
12
  llm = Llama(
13
  model_path=model_path,
14
- n_ctx=1024,
15
  n_threads=2,
16
- n_batch=512,
17
- offload_kqv=True
18
  )
19
 
20
  def respond(message, history):
 
2
  from llama_cpp import Llama
3
  from huggingface_hub import hf_hub_download
4
 
5
+ print("Loading 1.5B Model...")
6
+
7
  model_path = hf_hub_download(
8
+ repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
9
+ filename="qwen2.5-coder-1.5b-instruct-q8_0.gguf"
10
  )
11
 
 
12
  llm = Llama(
13
  model_path=model_path,
14
+ n_ctx=1024,
15
  n_threads=2,
16
+ n_batch=512
 
17
  )
18
 
19
  def respond(message, history):