Update app.py
Browse files
app.py
CHANGED
|
@@ -1,193 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
import io
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
from pathlib import Path
|
| 4 |
|
|
|
|
| 5 |
import gradio as gr
|
| 6 |
-
import
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
)
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
return "+0Hz"
|
| 72 |
-
|
| 73 |
-
# ===== 4. EXTERNAL STREAMING ENDPOINTS =====
|
| 74 |
-
|
| 75 |
-
# --- Direct Audio Stream (GET & POST) ---
|
| 76 |
-
@fastapi_app.api_route("/tts", methods=["GET", "POST"])
|
| 77 |
-
async def tts_stream(request: Request):
|
| 78 |
-
if request.method == "POST":
|
| 79 |
try:
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
except Exception:
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
text = data.get("text", "")
|
| 87 |
-
if not text:
|
| 88 |
-
return JSONResponse({"error": "Missing 'text' parameter"}, status_code=400)
|
| 89 |
-
|
| 90 |
-
voice = data.get("voice", "en-US-AriaNeural")
|
| 91 |
-
raw_rate = data.get("rate") or data.get("speed")
|
| 92 |
-
rate = format_rate(raw_rate)
|
| 93 |
-
pitch = format_pitch(data.get("pitch"))
|
| 94 |
-
|
| 95 |
-
async def generate_audio():
|
| 96 |
-
communicate = edge_tts.Communicate(text=text, voice=voice, rate=rate, pitch=pitch)
|
| 97 |
-
async for chunk in communicate.stream():
|
| 98 |
-
if chunk["type"] == "audio":
|
| 99 |
-
yield chunk["data"]
|
| 100 |
-
|
| 101 |
-
return StreamingResponse(
|
| 102 |
-
generate_audio(),
|
| 103 |
-
media_type="audio/mpeg",
|
| 104 |
-
headers={
|
| 105 |
-
"Cache-Control": "no-cache",
|
| 106 |
-
"Content-Disposition": "inline; filename=tts.mp3"
|
| 107 |
-
}
|
| 108 |
-
)
|
| 109 |
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
"
|
| 160 |
-
|
| 161 |
-
"
|
| 162 |
-
"zh-CN-XiaoxiaoNeural"
|
| 163 |
-
]
|
| 164 |
-
|
| 165 |
-
with gr.Blocks(title="High-Speed Edge-TTS API") as demo:
|
| 166 |
-
gr.Markdown(
|
| 167 |
-
f"# β‘ High-Speed Edge-TTS Server\n"
|
| 168 |
-
f"Running with {CORES} CPU Cores Allocated\n\n"
|
| 169 |
-
f"**External API Endpoints:**\n"
|
| 170 |
-
f"- `GET / POST /tts?text=...&voice=...&speed=1.0&pitch=+0Hz`\n"
|
| 171 |
-
f"- `POST /v1/audio/speech` (OpenAI Compatible)\n"
|
| 172 |
-
f"- `GET /tts/voices` (List All Available Edge-TTS Voices)"
|
| 173 |
-
)
|
| 174 |
with gr.Row():
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
I'm **Claude** (Anthropic). My reliable knowledge cutoff is **end of January 2025** β I know things up to around then; anything later (new llama.cpp release tags, brandβnew model repos like a "MiniCPM5") I can't verify, so I've made those **envβconfigurable + autoβdetected** in the code below instead of hardβcoding something that might 404.
|
| 2 |
+
|
| 3 |
+
---
|
| 4 |
+
|
| 5 |
+
# 1. `app.py`
|
| 6 |
+
|
| 7 |
+
```python
|
| 8 |
+
# app.py β GGUF chat Space (prebuilt llama.cpp server from GitHub, NO local compile)
|
| 9 |
import os
|
| 10 |
import io
|
| 11 |
+
import re
|
| 12 |
+
import json
|
| 13 |
+
import time
|
| 14 |
+
import stat
|
| 15 |
+
import zipfile
|
| 16 |
+
import atexit
|
| 17 |
+
import shutil
|
| 18 |
+
import signal
|
| 19 |
+
import socket
|
| 20 |
+
import subprocess
|
| 21 |
+
import urllib.request
|
| 22 |
from pathlib import Path
|
| 23 |
|
| 24 |
+
import requests
|
| 25 |
import gradio as gr
|
| 26 |
+
from huggingface_hub import hf_hub_download, list_repo_files
|
| 27 |
+
|
| 28 |
+
# ----------------------------------------------------------------------------
|
| 29 |
+
# 0. CONFIG (everything overridable with Space "Variables and secrets")
|
| 30 |
+
# ----------------------------------------------------------------------------
|
| 31 |
+
MODEL_REPO = os.getenv("MODEL_REPO", "openbmb/MiniCPM5-2B-GGUF") # <- verify it exists
|
| 32 |
+
MODEL_FILE = os.getenv("MODEL_FILE", "") # "" = auto-pick Q4_K_M
|
| 33 |
+
QUANT_HINT = os.getenv("QUANT_HINT", "Q4_K_M")
|
| 34 |
+
FALLBACK_REPO= os.getenv("FALLBACK_REPO","bartowski/MiniCPM3-4B-GGUF")
|
| 35 |
+
|
| 36 |
+
LLAMA_TAG = os.getenv("LLAMA_TAG", "latest") # pin e.g. "b4667" for reproducibility
|
| 37 |
+
LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
|
| 38 |
+
PORT = int(os.getenv("LLAMA_PORT", "8033"))
|
| 39 |
+
CTX = int(os.getenv("CTX", "4096"))
|
| 40 |
+
THREADS = int(os.getenv("THREADS", str(os.cpu_count() or 2)))
|
| 41 |
+
NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
|
| 42 |
+
|
| 43 |
+
WORK = Path(os.getenv("WORK_DIR", "/tmp/llamacpp"))
|
| 44 |
+
WORK.mkdir(parents=True, exist_ok=True)
|
| 45 |
+
|
| 46 |
+
# ----------------------------------------------------------------------------
|
| 47 |
+
# 1. GET A PREBUILT llama.cpp BINARY FROM GITHUB RELEASES (no cmake, no gcc)
|
| 48 |
+
# ----------------------------------------------------------------------------
|
| 49 |
+
def _gh_release(tag: str) -> dict:
|
| 50 |
+
url = ("https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
|
| 51 |
+
if tag == "latest" else
|
| 52 |
+
f"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{tag}")
|
| 53 |
+
req = urllib.request.Request(url, headers={"User-Agent": "hf-space"})
|
| 54 |
+
tok = os.getenv("GITHUB_TOKEN")
|
| 55 |
+
if tok:
|
| 56 |
+
req.add_header("Authorization", f"Bearer {tok}")
|
| 57 |
+
with urllib.request.urlopen(req, timeout=60) as r:
|
| 58 |
+
return json.load(r)
|
| 59 |
+
|
| 60 |
+
def ensure_llama_server() -> Path:
|
| 61 |
+
cached = next(iter(WORK.glob("**/llama-server")), None)
|
| 62 |
+
if cached:
|
| 63 |
+
return cached
|
| 64 |
+
|
| 65 |
+
rel = _gh_release(LLAMA_TAG)
|
| 66 |
+
bad = ("cuda", "hip", "vulkan", "sycl", "musa", "arm64", "macos", "win")
|
| 67 |
+
asset = next(
|
| 68 |
+
a for a in rel["assets"]
|
| 69 |
+
if a["name"].endswith(".zip")
|
| 70 |
+
and LLAMA_ASSET in a["name"]
|
| 71 |
+
and not any(b in a["name"].lower() for b in bad)
|
| 72 |
+
)
|
| 73 |
+
print(f"[llama.cpp] {rel['tag_name']} -> {asset['name']}")
|
| 74 |
+
|
| 75 |
+
zpath = WORK / asset["name"]
|
| 76 |
+
urllib.request.urlretrieve(asset["browser_download_url"], zpath)
|
| 77 |
+
with zipfile.ZipFile(zpath) as z:
|
| 78 |
+
z.extractall(WORK)
|
| 79 |
+
zpath.unlink(missing_ok=True)
|
| 80 |
+
|
| 81 |
+
server = next(WORK.glob("**/llama-server"))
|
| 82 |
+
for f in server.parent.iterdir(): # binaries + bundled .so
|
| 83 |
+
f.chmod(f.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR)
|
| 84 |
+
return server
|
| 85 |
+
|
| 86 |
+
# ----------------------------------------------------------------------------
|
| 87 |
+
# 2. GET THE GGUF FROM THE HUB
|
| 88 |
+
# ----------------------------------------------------------------------------
|
| 89 |
+
def ensure_model() -> str:
|
| 90 |
+
for repo in (MODEL_REPO, FALLBACK_REPO):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
try:
|
| 92 |
+
name = MODEL_FILE
|
| 93 |
+
if not name:
|
| 94 |
+
files = [f for f in list_repo_files(repo) if f.endswith(".gguf")]
|
| 95 |
+
cand = [f for f in files if QUANT_HINT.lower() in f.lower()] or files
|
| 96 |
+
# skip multi-part splits for simplicity
|
| 97 |
+
cand = [f for f in cand if "-00002-of-" not in f]
|
| 98 |
+
name = sorted(cand, key=len)[0]
|
| 99 |
+
print(f"[model] {repo}/{name}")
|
| 100 |
+
return hf_hub_download(repo_id=repo, filename=name,
|
| 101 |
+
local_dir=str(WORK / "models"))
|
| 102 |
+
except Exception as e:
|
| 103 |
+
print(f"[model] {repo} failed: {e}")
|
| 104 |
+
raise RuntimeError("No GGUF could be downloaded β set MODEL_REPO / MODEL_FILE.")
|
| 105 |
+
|
| 106 |
+
# ----------------------------------------------------------------------------
|
| 107 |
+
# 3. BOOT llama-server AS A SUBPROCESS (OpenAI-compatible endpoint on 127.0.0.1)
|
| 108 |
+
# ----------------------------------------------------------------------------
|
| 109 |
+
BASE = f"http://127.0.0.1:{PORT}"
|
| 110 |
+
PROC = None
|
| 111 |
+
|
| 112 |
+
def start_server():
|
| 113 |
+
global PROC
|
| 114 |
+
server = ensure_llama_server()
|
| 115 |
+
model = ensure_model()
|
| 116 |
+
|
| 117 |
+
env = os.environ.copy()
|
| 118 |
+
env["LD_LIBRARY_PATH"] = f"{server.parent}:{env.get('LD_LIBRARY_PATH','')}"
|
| 119 |
+
|
| 120 |
+
cmd = [str(server), "-m", model,
|
| 121 |
+
"--host", "127.0.0.1", "--port", str(PORT),
|
| 122 |
+
"-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
|
| 123 |
+
"--parallel", "2", "--cont-batching"]
|
| 124 |
+
print("[spawn]", " ".join(cmd))
|
| 125 |
+
PROC = subprocess.Popen(cmd, env=env,
|
| 126 |
+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
| 127 |
+
text=True, bufsize=1)
|
| 128 |
+
|
| 129 |
+
for _ in range(600): # up to ~5 min for first cold load
|
| 130 |
+
if PROC.poll() is not None:
|
| 131 |
+
raise RuntimeError("llama-server died:\n" + (PROC.stdout.read() or ""))
|
| 132 |
+
try:
|
| 133 |
+
if requests.get(BASE + "/health", timeout=2).status_code == 200:
|
| 134 |
+
print("[llama-server] ready")
|
| 135 |
+
return
|
| 136 |
except Exception:
|
| 137 |
+
pass
|
| 138 |
+
time.sleep(0.5)
|
| 139 |
+
raise RuntimeError("llama-server did not become healthy in time")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
+
@atexit.register
|
| 142 |
+
def _stop():
|
| 143 |
+
if PROC and PROC.poll() is None:
|
| 144 |
+
PROC.send_signal(signal.SIGTERM)
|
| 145 |
+
|
| 146 |
+
start_server()
|
| 147 |
+
|
| 148 |
+
# ----------------------------------------------------------------------------
|
| 149 |
+
# 4. INFERENCE (streaming, OpenAI schema)
|
| 150 |
+
# ----------------------------------------------------------------------------
|
| 151 |
+
def chat(message, history, system_prompt, max_tokens, temperature, top_p):
|
| 152 |
+
message = (message or "").strip()
|
| 153 |
+
if not message:
|
| 154 |
+
yield ""
|
| 155 |
+
return
|
| 156 |
+
|
| 157 |
+
msgs = [{"role": "system", "content": system_prompt}] if system_prompt else []
|
| 158 |
+
for h in (history or []): # gradio "messages" format
|
| 159 |
+
if isinstance(h, dict) and h.get("content"):
|
| 160 |
+
msgs.append({"role": h["role"], "content": h["content"]})
|
| 161 |
+
msgs.append({"role": "user", "content": message})
|
| 162 |
+
|
| 163 |
+
payload = {"messages": msgs, "stream": True,
|
| 164 |
+
"max_tokens": int(max_tokens),
|
| 165 |
+
"temperature": float(temperature),
|
| 166 |
+
"top_p": float(top_p)}
|
| 167 |
+
|
| 168 |
+
out = ""
|
| 169 |
+
with requests.post(BASE + "/v1/chat/completions", json=payload,
|
| 170 |
+
stream=True, timeout=900) as r:
|
| 171 |
+
r.raise_for_status()
|
| 172 |
+
for line in r.iter_lines(decode_unicode=True):
|
| 173 |
+
if not line or not line.startswith("data: "):
|
| 174 |
+
continue
|
| 175 |
+
chunk = line[6:]
|
| 176 |
+
if chunk.strip() == "[DONE]":
|
| 177 |
+
break
|
| 178 |
+
try:
|
| 179 |
+
delta = json.loads(chunk)["choices"][0]["delta"].get("content")
|
| 180 |
+
except Exception:
|
| 181 |
+
continue
|
| 182 |
+
if delta:
|
| 183 |
+
out += delta
|
| 184 |
+
yield out
|
| 185 |
+
|
| 186 |
+
# ----------------------------------------------------------------------------
|
| 187 |
+
# 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
|
| 188 |
+
# ----------------------------------------------------------------------------
|
| 189 |
+
with gr.Blocks(title="GGUF Chat (llama.cpp prebuilt)") as demo:
|
| 190 |
+
gr.Markdown(f"# π¦ GGUF Chat β `{MODEL_REPO}`\nCPU inference via prebuilt `llama-server`.")
|
| 191 |
+
|
| 192 |
+
system = gr.Textbox(label="System prompt", value="You are a helpful assistant.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 193 |
with gr.Row():
|
| 194 |
+
max_t = gr.Slider(16, 2048, value=512, step=16, label="Max new tokens")
|
| 195 |
+
temp = gr.Slider(0.0, 2.0, value=0.7, step=0.05, label="Temperature")
|
| 196 |
+
top_p = gr.Slider(0.1, 1.0, value=0.95, step=0.01, label="Top-p")
|
| 197 |
+
|
| 198 |
+
gr.ChatInterface(
|
| 199 |
+
fn=chat,
|
| 200 |
+
type="messages",
|
| 201 |
+
additional_inputs=[system, max_t, temp, top_p],
|
| 202 |
+
api_name="chat", # <- the external endpoint
|
| 203 |
+
examples=[["Explain GGUF quantization in two sentences."],
|
| 204 |
+
["Write a haiku about CPUs."]],
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
demo.queue(max_size=20).launch(server_name="0.0.0.0",
|
| 208 |
+
server_port=int(os.getenv("GRADIO_PORT", "7860")),
|
| 209 |
+
ssr_mode=False,
|
| 210 |
+
show_api=True)
|
| 211 |
+
```
|
| 212 |
+
|
| 213 |
+
---
|
| 214 |
+
|
| 215 |
+
# 2. `requirements.txt`
|
| 216 |
+
|
| 217 |
+
```
|
| 218 |
+
gradio==5.9.1
|
| 219 |
+
huggingface_hub>=0.26.0
|
| 220 |
+
requests>=2.32.0
|
| 221 |
+
```
|
| 222 |
+
|
| 223 |
+
That's it β **no `llama-cpp-python`, no compiler, no `spaces`**. The engine is a downloaded ELF binary.
|
| 224 |
+
|
| 225 |
+
> If you *insist* on the Python binding instead of the HTTP server, the only no-compile way is a prebuilt wheel index:
|
| 226 |
+
> ```
|
| 227 |
+
> --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
|
| 228 |
+
> llama-cpp-python
|
| 229 |
+
> ```
|
| 230 |
+
> (wheel availability depends on your Python version β the subprocess approach above has zero such risk).
|
| 231 |
+
|
| 232 |
+
---
|
| 233 |
+
|
| 234 |
+
# 3. Short doc β the important bits
|
| 235 |
+
|
| 236 |
+
### A. How the original Edge-TTS code "avoids GPU"
|
| 237 |
+
| Thing | What it actually does |
|
| 238 |
+
|---|---|
|
| 239 |
+
| `import spaces` + `@spaces.GPU(duration=1)` | On **ZeroGPU** hardware, HF statically scans the file for a `@spaces.GPU` decorator at startup. No decorator β the Space is rejected/ idles wrongly. The `dummy_gpu_trigger` is a **decoy** so the container boots. |
|
| 240 |
+
| `dummy_btn = gr.Button(visible=False)` | Keeps the decorated fn wired into the graph (so the scanner sees it used) while hiding it from users. |
|
| 241 |
+
| Real work (`edge_tts`) is pure network I/O | Microsoft's cloud does the synthesis β **0 GPU seconds consumed**, no CUDA, no torch. |
|
| 242 |
+
| `duration=1` | If ZeroGPU ever *does* allocate, it reserves 1 second β negligible quota. |
|
| 243 |
+
|
| 244 |
+
π **On a plain CPU Space you should delete `import spaces` and the dummy entirely** (that's what I did above). Keep the trick only if the Space hardware is literally ZeroGPU and you want it to boot while doing CPU/network work.
|
| 245 |
+
|
| 246 |
+
### B. How it becomes an external API
|
| 247 |
+
`api_name="speech"` / `api_name="chat"` publishes a named route. Everything else is Gradio's built-in HTTP layer.
|
| 248 |
+
|
| 249 |
+
**Python client**
|
| 250 |
+
```python
|
| 251 |
+
from gradio_client import Client
|
| 252 |
+
c = Client("USER/SPACE") # or "http://localhost:7860"
|
| 253 |
+
print(c.predict("Hello", [], "You are helpful.", 512, 0.7, 0.95, api_name="/chat"))
|
| 254 |
+
```
|
| 255 |
+
|
| 256 |
+
**Raw REST (Gradio 5 = 2-step)**
|
| 257 |
+
```bash
|
| 258 |
+
EV=$(curl -s -XPOST https://USER-SPACE.hf.space/gradio_api/call/chat \
|
| 259 |
+
-H 'Content-Type: application/json' \
|
| 260 |
+
-d '{"data":["Hello",[],"You are helpful.",512,0.7,0.95]}' | jq -r .event_id)
|
| 261 |
+
curl -N https://USER-SPACE.hf.space/gradio_api/call/chat/$EV # SSE stream
|
| 262 |
+
```
|
| 263 |
+
* Live schema: **`https://<space>.hf.space/?view=api`**
|
| 264 |
+
* Private Space β header `Authorization: Bearer hf_xxx`.
|
| 265 |
+
* `show_api=True` + `demo.queue()` are required for stable concurrent API use.
|
| 266 |
+
* Generator functions stream over the API too (SSE), same as in the UI.
|
| 267 |
+
|
| 268 |
+
### C. llama.cpp specifics in this app
|
| 269 |
+
* Binary: `ggml-org/llama.cpp` GitHub **Releases β `llama-<tag>-bin-ubuntu-x64.zip`** (CPU, glibc). Pin it: `LLAMA_TAG=b4667` β `latest` can break asset names.
|
| 270 |
+
* Bundled `libllama.so`/`libggml*.so` sit next to the binary β **`LD_LIBRARY_PATH` must include that dir** or you get "cannot open shared object file".
|
| 271 |
+
* `llama-server` speaks **OpenAI v1** (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`) + `/health`, `/props`. Chat template is read from the GGUF metadata β no manual prompt formatting.
|
| 272 |
+
* It stays on `127.0.0.1` β only Gradio is exposed. To expose it directly, mount it or proxy it (HF Spaces only routes **one** port, 7860).
|
| 273 |
+
|
| 274 |
+
### D. Resources / gotchas for a ~1.6 GB Q4_K_M
|
| 275 |
+
* RAM β model (1.56 GB) + KV-cache. KV at 4096 ctx for a 2B β 0.3β0.6 GB β fits free CPU Spaces (16 GB) easily; lower `CTX` if tight.
|
| 276 |
+
* Speed on free 2-vCPU: roughly **5β15 tok/s** for a 1β2B Q4. Set `THREADS` = physical cores; more threads than cores *hurts*.
|
| 277 |
+
* First boot downloads ~50 MB binary + model β use `WORK_DIR=/data` on persistent storage, else it re-downloads every cold start.
|
| 278 |
+
* `-ngl 0` is mandatory for the CPU build; `-ngl 99` only works if you swap `LLAMA_ASSET` to a CUDA asset **and** run on GPU hardware (then you also need the matching cudart zip).
|
| 279 |
+
* `MODEL_REPO="openbmb/MiniCPM5-2B-GGUF"` β I can't confirm that repo exists (post-cutoff for me). If it 404s the code auto-falls back; just set `MODEL_REPO`/`MODEL_FILE` to whatever you see on the Hub.
|