Nafiul Huda commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -3,45 +3,97 @@ import subprocess
|
|
| 3 |
import time
|
| 4 |
import requests
|
| 5 |
import json
|
|
|
|
|
|
|
| 6 |
import gradio as gr
|
| 7 |
import spaces
|
| 8 |
from huggingface_hub import hf_hub_download
|
| 9 |
|
| 10 |
# 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker
|
| 11 |
-
# without wasting runtime resources or intercepting user execution.
|
| 12 |
@spaces.GPU(duration=5)
|
| 13 |
def dummy_gpu():
|
| 14 |
print("ZeroGPU validation satisfied.")
|
| 15 |
return 1
|
| 16 |
|
| 17 |
-
# Automatically execute it once right away
|
| 18 |
dummy_gpu()
|
| 19 |
|
| 20 |
-
# 2. DOWNLOAD MODEL
|
| 21 |
-
def
|
| 22 |
print("Downloading model...")
|
| 23 |
repo = "Abiray/MiniCPM5-1B-GGUF"
|
| 24 |
filename = "minicpm5-1b-Q6_K.gguf"
|
| 25 |
model_path = hf_hub_download(repo_id=repo, filename=filename)
|
| 26 |
print(f"Model downloaded to: {model_path}")
|
|
|
|
| 27 |
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
else:
|
| 35 |
-
print("llama.cpp already compiled.")
|
| 36 |
|
| 37 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
-
#
|
| 40 |
-
def start_server(model_path):
|
| 41 |
-
# Using the exact parameters from your Dockerfile but pointing to local 8080 port
|
| 42 |
-
# so Gradio can occupy the main 7860 port safely.
|
| 43 |
cmd = [
|
| 44 |
-
|
| 45 |
"-m", model_path,
|
| 46 |
"--host", "127.0.0.1",
|
| 47 |
"--port", "8080",
|
|
@@ -55,11 +107,9 @@ def start_server(model_path):
|
|
| 55 |
print("Booting local llama-server with command:")
|
| 56 |
print(" ".join(cmd))
|
| 57 |
|
| 58 |
-
# Spawn server process in the background
|
| 59 |
server_process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
| 60 |
|
| 61 |
-
|
| 62 |
-
print("Waiting for llama-server to initialize (this may take a moment due to the 131k context KV cache allocation)...")
|
| 63 |
for _ in range(120):
|
| 64 |
try:
|
| 65 |
response = requests.get("http://127.0.0.1:8080/health")
|
|
@@ -71,12 +121,13 @@ def start_server(model_path):
|
|
| 71 |
|
| 72 |
raise RuntimeError("Server failed to start within the timeout period. You may be hitting an OOM error due to the 131072 context size.")
|
| 73 |
|
| 74 |
-
|
| 75 |
-
|
|
|
|
|
|
|
| 76 |
|
| 77 |
-
#
|
| 78 |
def chat_with_llama(message, history):
|
| 79 |
-
# Format history for OpenAI-compatible API
|
| 80 |
messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
|
| 81 |
for user_msg, assistant_msg in history:
|
| 82 |
messages.append({"role": "user", "content": user_msg})
|
|
@@ -117,10 +168,10 @@ def chat_with_llama(message, history):
|
|
| 117 |
except Exception as e:
|
| 118 |
yield f"Error communicating with local server: {str(e)}"
|
| 119 |
|
| 120 |
-
#
|
| 121 |
with gr.Blocks() as demo:
|
| 122 |
-
gr.Markdown("# Native CPU `llama.cpp`
|
| 123 |
-
gr.Markdown(f"Running **MiniCPM5-1B-GGUF** natively using `llama-server`
|
| 124 |
|
| 125 |
gr.ChatInterface(
|
| 126 |
fn=chat_with_llama,
|
|
|
|
| 3 |
import time
|
| 4 |
import requests
|
| 5 |
import json
|
| 6 |
+
import zipfile
|
| 7 |
+
import stat
|
| 8 |
import gradio as gr
|
| 9 |
import spaces
|
| 10 |
from huggingface_hub import hf_hub_download
|
| 11 |
|
| 12 |
# 1. DUMMY GPU FUNCTION: satisfies the Hugging Face startup checker
|
|
|
|
| 13 |
@spaces.GPU(duration=5)
|
| 14 |
def dummy_gpu():
|
| 15 |
print("ZeroGPU validation satisfied.")
|
| 16 |
return 1
|
| 17 |
|
| 18 |
+
# Automatically execute it once right away
|
| 19 |
dummy_gpu()
|
| 20 |
|
| 21 |
+
# 2. DOWNLOAD MODEL
|
| 22 |
+
def setup_model():
|
| 23 |
print("Downloading model...")
|
| 24 |
repo = "Abiray/MiniCPM5-1B-GGUF"
|
| 25 |
filename = "minicpm5-1b-Q6_K.gguf"
|
| 26 |
model_path = hf_hub_download(repo_id=repo, filename=filename)
|
| 27 |
print(f"Model downloaded to: {model_path}")
|
| 28 |
+
return model_path
|
| 29 |
|
| 30 |
+
# 3. DOWNLOAD PRE-BUILT LLAMA.CPP BINARY
|
| 31 |
+
def get_prebuilt_llama():
|
| 32 |
+
server_bin = "./llama-server"
|
| 33 |
+
if os.path.exists(server_bin):
|
| 34 |
+
print("Pre-built llama-server already exists.")
|
| 35 |
+
return server_bin
|
|
|
|
|
|
|
| 36 |
|
| 37 |
+
print("Fetching latest pre-built llama.cpp release...")
|
| 38 |
+
api_url = "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
|
| 39 |
+
|
| 40 |
+
try:
|
| 41 |
+
resp = requests.get(api_url).json()
|
| 42 |
+
target_asset = None
|
| 43 |
+
# Look specifically for the raw CPU Ubuntu x64 binary
|
| 44 |
+
for asset in resp.get("assets", []):
|
| 45 |
+
name = asset["name"].lower()
|
| 46 |
+
if "ubuntu-x64" in name and name.endswith(".zip") and not any(x in name for x in ["vulkan", "rocm", "sycl", "openvino", "cuda"]):
|
| 47 |
+
target_asset = asset
|
| 48 |
+
break
|
| 49 |
+
|
| 50 |
+
if not target_asset:
|
| 51 |
+
raise ValueError("No matching CPU asset found in latest release.")
|
| 52 |
+
|
| 53 |
+
download_url = target_asset["browser_download_url"]
|
| 54 |
+
zip_name = target_asset["name"]
|
| 55 |
+
|
| 56 |
+
except Exception as e:
|
| 57 |
+
print(f"GitHub API check failed ({e}), using fallback direct URL...")
|
| 58 |
+
# Safe fallback in case Hugging Face hits GitHub's unauthenticated API limit
|
| 59 |
+
download_url = "https://github.com/ggerganov/llama.cpp/releases/download/b3400/llama-b3400-bin-ubuntu-x64.zip"
|
| 60 |
+
zip_name = "llama-fallback.zip"
|
| 61 |
+
|
| 62 |
+
print(f"Downloading {zip_name} from {download_url}...")
|
| 63 |
+
with requests.get(download_url, stream=True) as r:
|
| 64 |
+
r.raise_for_status()
|
| 65 |
+
with open(zip_name, "wb") as f:
|
| 66 |
+
for chunk in r.iter_content(chunk_size=8192):
|
| 67 |
+
f.write(chunk)
|
| 68 |
+
|
| 69 |
+
print("Extracting zip...")
|
| 70 |
+
with zipfile.ZipFile(zip_name, 'r') as zip_ref:
|
| 71 |
+
zip_ref.extractall("./llama_extracted")
|
| 72 |
+
|
| 73 |
+
# Locate the binary inside the extracted folder
|
| 74 |
+
found_bin = None
|
| 75 |
+
for root, dirs, files in os.walk("./llama_extracted"):
|
| 76 |
+
if "llama-server" in files:
|
| 77 |
+
found_bin = os.path.join(root, "llama-server")
|
| 78 |
+
break
|
| 79 |
+
|
| 80 |
+
if not found_bin:
|
| 81 |
+
raise RuntimeError("llama-server binary not found in the extracted zip!")
|
| 82 |
+
|
| 83 |
+
# Move it to root and clean up
|
| 84 |
+
os.rename(found_bin, server_bin)
|
| 85 |
+
|
| 86 |
+
# Make it executable (chmod +x)
|
| 87 |
+
st = os.stat(server_bin)
|
| 88 |
+
os.chmod(server_bin, st.st_mode | stat.S_IEXEC)
|
| 89 |
+
|
| 90 |
+
print("Pre-built llama-server is ready!")
|
| 91 |
+
return server_bin
|
| 92 |
|
| 93 |
+
# 4. START BACKGROUND SERVER
|
| 94 |
+
def start_server(model_path, server_bin_path):
|
|
|
|
|
|
|
| 95 |
cmd = [
|
| 96 |
+
server_bin_path,
|
| 97 |
"-m", model_path,
|
| 98 |
"--host", "127.0.0.1",
|
| 99 |
"--port", "8080",
|
|
|
|
| 107 |
print("Booting local llama-server with command:")
|
| 108 |
print(" ".join(cmd))
|
| 109 |
|
|
|
|
| 110 |
server_process = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
|
| 111 |
|
| 112 |
+
print("Waiting for llama-server to initialize (allocating 131k context cache)...")
|
|
|
|
| 113 |
for _ in range(120):
|
| 114 |
try:
|
| 115 |
response = requests.get("http://127.0.0.1:8080/health")
|
|
|
|
| 121 |
|
| 122 |
raise RuntimeError("Server failed to start within the timeout period. You may be hitting an OOM error due to the 131072 context size.")
|
| 123 |
|
| 124 |
+
# Execute initialization
|
| 125 |
+
model_filepath = setup_model()
|
| 126 |
+
llama_executable = get_prebuilt_llama()
|
| 127 |
+
start_server(model_filepath, llama_executable)
|
| 128 |
|
| 129 |
+
# 5. GRADIO UI TO INTERACT WITH LOCAL SERVER
|
| 130 |
def chat_with_llama(message, history):
|
|
|
|
| 131 |
messages = [{"role": "system", "content": "You are a helpful AI assistant."}]
|
| 132 |
for user_msg, assistant_msg in history:
|
| 133 |
messages.append({"role": "user", "content": user_msg})
|
|
|
|
| 168 |
except Exception as e:
|
| 169 |
yield f"Error communicating with local server: {str(e)}"
|
| 170 |
|
| 171 |
+
# 6. LAUNCH GRADIO APP
|
| 172 |
with gr.Blocks() as demo:
|
| 173 |
+
gr.Markdown("# Native CPU Pre-Built `llama.cpp` on ZeroGPU Space")
|
| 174 |
+
gr.Markdown(f"Running **MiniCPM5-1B-GGUF** natively using the pre-compiled Ubuntu `llama-server`! Bypassing compilation and GPU logic entirely.")
|
| 175 |
|
| 176 |
gr.ChatInterface(
|
| 177 |
fn=chat_with_llama,
|