Nafiul Huda commited on
Commit
8e021d6
·
verified ·
1 Parent(s): b291c63

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +79 -28
app.py CHANGED
@@ -3,45 +3,97 @@ import subprocess
3
  import time
4
  import requests
5
  import json
 
 
6
  import gradio as gr
7
  import spaces
8
  from huggingface_hub import hf_hub_download
9
 
10
  # 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker
11
- # without wasting runtime resources or intercepting user execution.
12
  @spaces.GPU(duration=5)
13
  def dummy_gpu():
14
  print("ZeroGPU validation satisfied.")
15
  return 1
16
 
17
- # Automatically execute it once right away during script interpretation
18
  dummy_gpu()
19
 
20
- # 2. DOWNLOAD MODEL & COMPILE LLAMA.CPP SERVER
21
- def setup_environment():
22
  print("Downloading model...")
23
  repo = "Abiray/MiniCPM5-1B-GGUF"
24
  filename = "minicpm5-1b-Q6_K.gguf"
25
  model_path = hf_hub_download(repo_id=repo, filename=filename)
26
  print(f"Model downloaded to: {model_path}")
 
27
 
28
- if not os.path.exists("llama.cpp/llama-server"):
29
- print("llama-server binary not found. Cloning and compiling llama.cpp...")
30
- subprocess.run("git clone https://github.com/ggerganov/llama.cpp.git", shell=True)
31
- # Compile only the server binary to save time
32
- subprocess.run("cd llama.cpp && make -j4 llama-server", shell=True)
33
- print("Compilation finished!")
34
- else:
35
- print("llama.cpp already compiled.")
36
 
37
- return model_path
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
38
 
39
- # 3. START BACKGROUND SERVER
40
- def start_server(model_path):
41
- # Using the exact parameters from your Dockerfile but pointing to local 8080 port
42
- # so Gradio can occupy the main 7860 port safely.
43
  cmd = [
44
- "./llama.cpp/llama-server",
45
  "-m", model_path,
46
  "--host", "127.0.0.1",
47
  "--port", "8080",
@@ -55,11 +107,9 @@ def start_server(model_path):
55
  print("Booting local llama-server with command:")
56
  print(" ".join(cmd))
57
 
58
- # Spawn server process in the background
59
  server_process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
60
 
61
- # Wait for the server to be healthy
62
- print("Waiting for llama-server to initialize (this may take a moment due to the 131k context KV cache allocation)...")
63
  for _ in range(120):
64
  try:
65
  response = requests.get("http://127.0.0.1:8080/health")
@@ -71,12 +121,13 @@ def start_server(model_path):
71
 
72
  raise RuntimeError("Server failed to start within the timeout period. You may be hitting an OOM error due to the 131072 context size.")
73
 
74
- model_filepath = setup_environment()
75
- start_server(model_filepath)
 
 
76
 
77
- # 4. GRADIO UI TO INTERACT WITH LOCAL SERVER
78
  def chat_with_llama(message, history):
79
- # Format history for OpenAI-compatible API
80
  messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
81
  for user_msg, assistant_msg in history:
82
  messages.append({"role": "user", "content": user_msg})
@@ -117,10 +168,10 @@ def chat_with_llama(message, history):
117
  except Exception as e:
118
  yield f"Error communicating with local server: {str(e)}"
119
 
120
- # 5. LAUNCH GRADIO APP
121
  with gr.Blocks() as demo:
122
- gr.Markdown("# Native CPU `llama.cpp` Execution on ZeroGPU Space")
123
- gr.Markdown(f"Running **MiniCPM5-1B-GGUF** natively using `llama-server` behind the scenes! Bypassing GPU logic entirely.")
124
 
125
  gr.ChatInterface(
126
  fn=chat_with_llama,
 
3
  import time
4
  import requests
5
  import json
6
+ import zipfile
7
+ import stat
8
  import gradio as gr
9
  import spaces
10
  from huggingface_hub import hf_hub_download
11
 
12
  # 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker
 
13
  @spaces.GPU(duration=5)
14
  def dummy_gpu():
15
  print("ZeroGPU validation satisfied.")
16
  return 1
17
 
18
+ # Automatically execute it once right away
19
  dummy_gpu()
20
 
21
+ # 2. DOWNLOAD MODEL
22
+ def setup_model():
23
  print("Downloading model...")
24
  repo = "Abiray/MiniCPM5-1B-GGUF"
25
  filename = "minicpm5-1b-Q6_K.gguf"
26
  model_path = hf_hub_download(repo_id=repo, filename=filename)
27
  print(f"Model downloaded to: {model_path}")
28
+ return model_path
29
 
30
+ # 3. DOWNLOAD PRE-BUILT LLAMA.CPP BINARY
31
+ def get_prebuilt_llama():
32
+ server_bin = "./llama-server"
33
+ if os.path.exists(server_bin):
34
+ print("Pre-built llama-server already exists.")
35
+ return server_bin
 
 
36
 
37
+ print("Fetching latest pre-built llama.cpp release...")
38
+ api_url = "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
39
+
40
+ try:
41
+ resp = requests.get(api_url).json()
42
+ target_asset = None
43
+ # Look specifically for the raw CPU Ubuntu x64 binary
44
+ for asset in resp.get("assets", []):
45
+ name = asset["name"].lower()
46
+ if "ubuntu-x64" in name and name.endswith(".zip") and not any(x in name for x in ["vulkan", "rocm", "sycl", "openvino", "cuda"]):
47
+ target_asset = asset
48
+ break
49
+
50
+ if not target_asset:
51
+ raise ValueError("No matching CPU asset found in latest release.")
52
+
53
+ download_url = target_asset["browser_download_url"]
54
+ zip_name = target_asset["name"]
55
+
56
+ except Exception as e:
57
+ print(f"GitHub API check failed ({e}), using fallback direct URL...")
58
+ # Safe fallback in case Hugging Face hits GitHub's unauthenticated API limit
59
+ download_url = "https://github.com/ggerganov/llama.cpp/releases/download/b3400/llama-b3400-bin-ubuntu-x64.zip"
60
+ zip_name = "llama-fallback.zip"
61
+
62
+ print(f"Downloading {zip_name} from {download_url}...")
63
+ with requests.get(download_url, stream=True) as r:
64
+ r.raise_for_status()
65
+ with open(zip_name, "wb") as f:
66
+ for chunk in r.iter_content(chunk_size=8192):
67
+ f.write(chunk)
68
+
69
+ print("Extracting zip...")
70
+ with zipfile.ZipFile(zip_name, 'r') as zip_ref:
71
+ zip_ref.extractall("./llama_extracted")
72
+
73
+ # Locate the binary inside the extracted folder
74
+ found_bin = None
75
+ for root, dirs, files in os.walk("./llama_extracted"):
76
+ if "llama-server" in files:
77
+ found_bin = os.path.join(root, "llama-server")
78
+ break
79
+
80
+ if not found_bin:
81
+ raise RuntimeError("llama-server binary not found in the extracted zip!")
82
+
83
+ # Move it to root and clean up
84
+ os.rename(found_bin, server_bin)
85
+
86
+ # Make it executable (chmod +x)
87
+ st = os.stat(server_bin)
88
+ os.chmod(server_bin, st.st_mode | stat.S_IEXEC)
89
+
90
+ print("Pre-built llama-server is ready!")
91
+ return server_bin
92
 
93
+ # 4. START BACKGROUND SERVER
94
+ def start_server(model_path, server_bin_path):
 
 
95
  cmd = [
96
+ server_bin_path,
97
  "-m", model_path,
98
  "--host", "127.0.0.1",
99
  "--port", "8080",
 
107
  print("Booting local llama-server with command:")
108
  print(" ".join(cmd))
109
 
 
110
  server_process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
111
 
112
+ print("Waiting for llama-server to initialize (allocating 131k context cache)...")
 
113
  for _ in range(120):
114
  try:
115
  response = requests.get("http://127.0.0.1:8080/health")
 
121
 
122
  raise RuntimeError("Server failed to start within the timeout period. You may be hitting an OOM error due to the 131072 context size.")
123
 
124
+ # Execute initialization
125
+ model_filepath = setup_model()
126
+ llama_executable = get_prebuilt_llama()
127
+ start_server(model_filepath, llama_executable)
128
 
129
+ # 5. GRADIO UI TO INTERACT WITH LOCAL SERVER
130
  def chat_with_llama(message, history):
 
131
  messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
132
  for user_msg, assistant_msg in history:
133
  messages.append({"role": "user", "content": user_msg})
 
168
  except Exception as e:
169
  yield f"Error communicating with local server: {str(e)}"
170
 
171
+ # 6. LAUNCH GRADIO APP
172
  with gr.Blocks() as demo:
173
+ gr.Markdown("# Native CPU Pre-Built `llama.cpp` on ZeroGPU Space")
174
+ gr.Markdown(f"Running **MiniCPM5-1B-GGUF** natively using the pre-compiled Ubuntu `llama-server`! Bypassing compilation and GPU logic entirely.")
175
 
176
  gr.ChatInterface(
177
  fn=chat_with_llama,