Nafiul Huda commited on
Commit
94e4f66
·
verified ·
1 Parent(s): 864d76b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +21 -16
app.py CHANGED
@@ -2,27 +2,33 @@ import os
2
  import gradio as gr
3
  import spaces
4
  from huggingface_hub import hf_hub_download
5
- from llama_cpp import Llama
6
 
7
- # 1. Download GGUF model from Hub during startup
 
 
 
8
  print("Downloading model...")
9
  model_path = hf_hub_download(
10
  repo_id="Abiray/MiniCPM5-1B-GGUF",
11
  filename="minicpm5-1b-Q6_K.gguf"
12
  )
13
 
14
- # 2. Initialize Llama.cpp with GPU layers enabled (n_gpu_layers=-1 sends all layers to GPU)
15
- print("Initializing model on GPU paths...")
16
- llm = Llama(
17
- model_path=model_path,
18
- n_ctx=2048,
19
- n_gpu_layers=-1
20
- )
21
-
22
- # 3. Use the spaces decorator to temporarily claim the ZeroGPU instance for inference
23
  @spaces.GPU(duration=60)
24
  def generate_response(message, history):
25
- # Construct an OpenAI/ChatML style prompt framework for MiniCPM
 
 
 
 
 
 
 
 
 
 
 
 
26
  prompt = f"<|user|>\n{message}\n<|assistant|>\n"
27
 
28
  # Run high-speed GPU accelerated inference
@@ -33,11 +39,10 @@ def generate_response(message, history):
33
  echo=False
34
  )
35
 
36
- response_text = output["choices"][0]["text"].strip()
37
- return response_text
38
 
39
- # 4. Minimal, high-performance Gradio interface
40
- with gr.Blocks(theme=gr.themes.Default(spacing_size="sm", radius_size="none")) as demo:
41
  gr.Markdown("### MiniCPM-1B GGUF ZeroGPU Pipeline")
42
  chatbot = gr.ChatInterface(
43
  fn=generate_response,
 
2
  import gradio as gr
3
  import spaces
4
  from huggingface_hub import hf_hub_download
 
5
 
6
+ # Global variable to hold our model instance once loaded
7
+ llm = None
8
+
9
+ # 1. Download model at root (this is safe, it's just fetching a file)
10
  print("Downloading model...")
11
  model_path = hf_hub_download(
12
  repo_id="Abiray/MiniCPM5-1B-GGUF",
13
  filename="minicpm5-1b-Q6_K.gguf"
14
  )
15
 
16
+ # 2. Wrap initialization inside the GPU context
 
 
 
 
 
 
 
 
17
  @spaces.GPU(duration=60)
18
  def generate_response(message, history):
19
+ global llm
20
+
21
+ # Lazy import/load only when the GPU is actually active and attached
22
+ if llm is None:
23
+ print("GPU attached! Loading llama_cpp and initializing model...")
24
+ from llama_cpp import Llama
25
+ llm = Llama(
26
+ model_path=model_path,
27
+ n_ctx=2048,
28
+ n_gpu_layers=-1 # Safely sends layers to the active GPU
29
+ )
30
+
31
+ # Construct prompt format
32
  prompt = f"<|user|>\n{message}\n<|assistant|>\n"
33
 
34
  # Run high-speed GPU accelerated inference
 
39
  echo=False
40
  )
41
 
42
+ return output["choices"][0]["text"].strip()
 
43
 
44
+ # 3. Gradio Interface
45
+ with gr.Blocks() as demo:
46
  gr.Markdown("### MiniCPM-1B GGUF ZeroGPU Pipeline")
47
  chatbot = gr.ChatInterface(
48
  fn=generate_response,