Unmid commited on
Commit
6a4954d
Β·
verified Β·
1 Parent(s): c96ef39

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +270 -184
app.py CHANGED
@@ -1,193 +1,279 @@
 
 
 
 
 
 
 
 
1
  import os
2
  import io
 
 
 
 
 
 
 
 
 
 
 
3
  from pathlib import Path
4
 
 
5
  import gradio as gr
6
- import edge_tts
7
- from fastapi import FastAPI, Request
8
- from fastapi.middleware.cors import CORSMiddleware
9
- from fastapi.responses import StreamingResponse, JSONResponse
10
-
11
- # ===== 1. CPU / SYSTEM RESOURCE DETECTOR =====
12
- def effective_cpus() -> int:
13
- try:
14
- quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()[:2]
15
- if quota != "max":
16
- return max(1, int(quota) // int(period))
17
- except Exception:
18
- pass
19
- try:
20
- return len(os.sched_getaffinity(0))
21
- except Exception:
22
- return os.cpu_count() or 2
23
-
24
- def memory_limit_gb():
25
- try:
26
- v = Path("/sys/fs/cgroup/memory.max").read_text().strip()
27
- if v != "max":
28
- return round(int(v) / 1e9, 1)
29
- except Exception:
30
- pass
31
- return None
32
-
33
- CORES = effective_cpus()
34
- RAM = memory_limit_gb() or "?"
35
- print(f"[resources] Effective cores: {CORES} | RAM limit: {RAM} GB")
36
-
37
- # ===== 2. FASTAPI SERVER & CORS SETUP =====
38
- fastapi_app = FastAPI(title="Edge-TTS-Server")
39
- fastapi_app.add_middleware(
40
- CORSMiddleware,
41
- allow_origins=["*"],
42
- allow_credentials=True,
43
- allow_methods=["*"],
44
- allow_headers=["*"],
45
- )
46
-
47
- # ===== 3. PARAMETER NORMALIZERS =====
48
- def format_rate(rate_input):
49
- if rate_input is None:
50
- return "+0%"
51
- val = str(rate_input).strip()
52
- if val.endswith("%"):
53
- return val if val.startswith(("+", "-")) else f"+{val}"
54
- try:
55
- f = float(val)
56
- pct = int(round((f - 1.0) * 100))
57
- return f"+{pct}%" if pct >= 0 else f"{pct}%"
58
- except ValueError:
59
- return "+0%"
60
-
61
- def format_pitch(pitch_input):
62
- if pitch_input is None:
63
- return "+0Hz"
64
- val = str(pitch_input).strip()
65
- if val.endswith("Hz") or val.endswith("%"):
66
- return val if val.startswith(("+", "-")) else f"+{val}"
67
- try:
68
- val_int = int(val)
69
- return f"+{val_int}Hz" if val_int >= 0 else f"{val_int}Hz"
70
- except ValueError:
71
- return "+0Hz"
72
-
73
- # ===== 4. EXTERNAL STREAMING ENDPOINTS =====
74
-
75
- # --- Direct Audio Stream (GET & POST) ---
76
- @fastapi_app.api_route("/tts", methods=["GET", "POST"])
77
- async def tts_stream(request: Request):
78
- if request.method == "POST":
79
  try:
80
- data = await request.json()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  except Exception:
82
- data = {}
83
- else:
84
- data = dict(request.query_params)
85
-
86
- text = data.get("text", "")
87
- if not text:
88
- return JSONResponse({"error": "Missing 'text' parameter"}, status_code=400)
89
-
90
- voice = data.get("voice", "en-US-AriaNeural")
91
- raw_rate = data.get("rate") or data.get("speed")
92
- rate = format_rate(raw_rate)
93
- pitch = format_pitch(data.get("pitch"))
94
-
95
- async def generate_audio():
96
- communicate = edge_tts.Communicate(text=text, voice=voice, rate=rate, pitch=pitch)
97
- async for chunk in communicate.stream():
98
- if chunk["type"] == "audio":
99
- yield chunk["data"]
100
-
101
- return StreamingResponse(
102
- generate_audio(),
103
- media_type="audio/mpeg",
104
- headers={
105
- "Cache-Control": "no-cache",
106
- "Content-Disposition": "inline; filename=tts.mp3"
107
- }
108
- )
109
 
110
- # --- OpenAI Audio Speech Compatible Endpoint ---
111
- @fastapi_app.api_route("/v1/audio/speech", methods=["POST"])
112
- async def openai_speech(request: Request):
113
- try:
114
- data = await request.json()
115
- except Exception:
116
- data = {}
117
-
118
- text = data.get("input", "")
119
- voice = data.get("voice", "en-US-AriaNeural")
120
- speed = data.get("speed", 1.0)
121
- rate = format_rate(speed)
122
-
123
- async def generate_audio():
124
- communicate = edge_tts.Communicate(text=text, voice=voice, rate=rate)
125
- async for chunk in communicate.stream():
126
- if chunk["type"] == "audio":
127
- yield chunk["data"]
128
-
129
- return StreamingResponse(generate_audio(), media_type="audio/mpeg")
130
-
131
- # --- Get All Available Voices ---
132
- @fastapi_app.get("/tts/voices")
133
- async def list_voices():
134
- voices = await edge_tts.list_voices()
135
- return JSONResponse(voices)
136
-
137
- # ===== 5. GRADIO TEST INTERFACE =====
138
- async def gradio_tts(text, voice, speed, pitch):
139
- if not text:
140
- return None
141
- rate_str = format_rate(speed)
142
- pitch_str = format_pitch(pitch)
143
- communicate = edge_tts.Communicate(text=text, voice=voice, rate=rate_str, pitch=pitch_str)
144
-
145
- buf = io.BytesIO()
146
- async for chunk in communicate.stream():
147
- if chunk["type"] == "audio":
148
- buf.write(chunk["data"])
149
- buf.seek(0)
150
- return buf.getvalue()
151
-
152
- DEFAULT_VOICES = [
153
- "en-US-AriaNeural",
154
- "en-US-ChristopherNeural",
155
- "en-US-GuyNeural",
156
- "en-US-JennyNeural",
157
- "en-GB-SoniaNeural",
158
- "en-GB-RyanNeural",
159
- "es-ES-AlvaroNeural",
160
- "fr-FR-DeniseNeural",
161
- "de-DE-KatjaNeural",
162
- "zh-CN-XiaoxiaoNeural"
163
- ]
164
-
165
- with gr.Blocks(title="High-Speed Edge-TTS API") as demo:
166
- gr.Markdown(
167
- f"# ⚑ High-Speed Edge-TTS Server\n"
168
- f"Running with {CORES} CPU Cores Allocated\n\n"
169
- f"**External API Endpoints:**\n"
170
- f"- `GET / POST /tts?text=...&voice=...&speed=1.0&pitch=+0Hz`\n"
171
- f"- `POST /v1/audio/speech` (OpenAI Compatible)\n"
172
- f"- `GET /tts/voices` (List All Available Edge-TTS Voices)"
173
- )
174
  with gr.Row():
175
- with gr.Column():
176
- text_input = gr.Textbox(label="Text", value="Hello! This is a real-time streaming test of Edge TTS.", lines=3)
177
- voice_dropdown = gr.Dropdown(choices=DEFAULT_VOICES, value="en-US-AriaNeural", label="Voice")
178
- speed_slider = gr.Slider(minimum=0.5, maximum=2.0, value=1.0, step=0.1, label="Speed / Rate")
179
- pitch_input = gr.Textbox(value="+0Hz", label="Pitch (e.g. +0Hz, +5Hz, -5Hz)")
180
- btn = gr.Button("Generate Speech", variant="primary")
181
- with gr.Column():
182
- audio_output = gr.Audio(label="Audio Output", autoplay=True)
183
-
184
- btn.click(fn=gradio_tts, inputs=[text_input, voice_dropdown, speed_slider, pitch_input], outputs=audio_output)
185
-
186
- # ===== 6. MOUNT GRADIO ON FASTAPI & EXPORT =====
187
- app = gr.mount_gradio_app(fastapi_app, demo, path="/")
188
-
189
- if __name__ == "__main__":
190
- import uvicorn
191
- # Only run uvicorn locally when app.py is run directly
192
- port = int(os.environ.get("PORT", 7860))
193
- uvicorn.run("app:app", host="0.0.0.0", port=port, reload=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ I'm **Claude** (Anthropic). My reliable knowledge cutoff is **end of January 2025** β€” I know things up to around then; anything later (new llama.cpp release tags, brand‑new model repos like a "MiniCPM5") I can't verify, so I've made those **env‑configurable + auto‑detected** in the code below instead of hard‑coding something that might 404.
2
+
3
+ ---
4
+
5
+ # 1. `app.py`
6
+
7
+ ```python
8
+ # app.py β€” GGUF chat Space (prebuilt llama.cpp server from GitHub, NO local compile)
9
  import os
10
  import io
11
+ import re
12
+ import json
13
+ import time
14
+ import stat
15
+ import zipfile
16
+ import atexit
17
+ import shutil
18
+ import signal
19
+ import socket
20
+ import subprocess
21
+ import urllib.request
22
  from pathlib import Path
23
 
24
+ import requests
25
  import gradio as gr
26
+ from huggingface_hub import hf_hub_download, list_repo_files
27
+
28
+ # ----------------------------------------------------------------------------
29
+ # 0. CONFIG (everything overridable with Space "Variables and secrets")
30
+ # ----------------------------------------------------------------------------
31
+ MODEL_REPO = os.getenv("MODEL_REPO", "openbmb/MiniCPM5-2B-GGUF") # <- verify it exists
32
+ MODEL_FILE = os.getenv("MODEL_FILE", "") # "" = auto-pick Q4_K_M
33
+ QUANT_HINT = os.getenv("QUANT_HINT", "Q4_K_M")
34
+ FALLBACK_REPO= os.getenv("FALLBACK_REPO","bartowski/MiniCPM3-4B-GGUF")
35
+
36
+ LLAMA_TAG = os.getenv("LLAMA_TAG", "latest") # pin e.g. "b4667" for reproducibility
37
+ LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
38
+ PORT = int(os.getenv("LLAMA_PORT", "8033"))
39
+ CTX = int(os.getenv("CTX", "4096"))
40
+ THREADS = int(os.getenv("THREADS", str(os.cpu_count() or 2)))
41
+ NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
42
+
43
+ WORK = Path(os.getenv("WORK_DIR", "/tmp/llamacpp"))
44
+ WORK.mkdir(parents=True, exist_ok=True)
45
+
46
+ # ----------------------------------------------------------------------------
47
+ # 1. GET A PREBUILT llama.cpp BINARY FROM GITHUB RELEASES (no cmake, no gcc)
48
+ # ----------------------------------------------------------------------------
49
+ def _gh_release(tag: str) -> dict:
50
+ url = ("https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
51
+ if tag == "latest" else
52
+ f"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{tag}")
53
+ req = urllib.request.Request(url, headers={"User-Agent": "hf-space"})
54
+ tok = os.getenv("GITHUB_TOKEN")
55
+ if tok:
56
+ req.add_header("Authorization", f"Bearer {tok}")
57
+ with urllib.request.urlopen(req, timeout=60) as r:
58
+ return json.load(r)
59
+
60
+ def ensure_llama_server() -> Path:
61
+ cached = next(iter(WORK.glob("**/llama-server")), None)
62
+ if cached:
63
+ return cached
64
+
65
+ rel = _gh_release(LLAMA_TAG)
66
+ bad = ("cuda", "hip", "vulkan", "sycl", "musa", "arm64", "macos", "win")
67
+ asset = next(
68
+ a for a in rel["assets"]
69
+ if a["name"].endswith(".zip")
70
+ and LLAMA_ASSET in a["name"]
71
+ and not any(b in a["name"].lower() for b in bad)
72
+ )
73
+ print(f"[llama.cpp] {rel['tag_name']} -> {asset['name']}")
74
+
75
+ zpath = WORK / asset["name"]
76
+ urllib.request.urlretrieve(asset["browser_download_url"], zpath)
77
+ with zipfile.ZipFile(zpath) as z:
78
+ z.extractall(WORK)
79
+ zpath.unlink(missing_ok=True)
80
+
81
+ server = next(WORK.glob("**/llama-server"))
82
+ for f in server.parent.iterdir(): # binaries + bundled .so
83
+ f.chmod(f.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR)
84
+ return server
85
+
86
+ # ----------------------------------------------------------------------------
87
+ # 2. GET THE GGUF FROM THE HUB
88
+ # ----------------------------------------------------------------------------
89
+ def ensure_model() -> str:
90
+ for repo in (MODEL_REPO, FALLBACK_REPO):
 
 
 
 
 
 
 
 
91
  try:
92
+ name = MODEL_FILE
93
+ if not name:
94
+ files = [f for f in list_repo_files(repo) if f.endswith(".gguf")]
95
+ cand = [f for f in files if QUANT_HINT.lower() in f.lower()] or files
96
+ # skip multi-part splits for simplicity
97
+ cand = [f for f in cand if "-00002-of-" not in f]
98
+ name = sorted(cand, key=len)[0]
99
+ print(f"[model] {repo}/{name}")
100
+ return hf_hub_download(repo_id=repo, filename=name,
101
+ local_dir=str(WORK / "models"))
102
+ except Exception as e:
103
+ print(f"[model] {repo} failed: {e}")
104
+ raise RuntimeError("No GGUF could be downloaded β€” set MODEL_REPO / MODEL_FILE.")
105
+
106
+ # ----------------------------------------------------------------------------
107
+ # 3. BOOT llama-server AS A SUBPROCESS (OpenAI-compatible endpoint on 127.0.0.1)
108
+ # ----------------------------------------------------------------------------
109
+ BASE = f"http://127.0.0.1:{PORT}"
110
+ PROC = None
111
+
112
+ def start_server():
113
+ global PROC
114
+ server = ensure_llama_server()
115
+ model = ensure_model()
116
+
117
+ env = os.environ.copy()
118
+ env["LD_LIBRARY_PATH"] = f"{server.parent}:{env.get('LD_LIBRARY_PATH','')}"
119
+
120
+ cmd = [str(server), "-m", model,
121
+ "--host", "127.0.0.1", "--port", str(PORT),
122
+ "-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
123
+ "--parallel", "2", "--cont-batching"]
124
+ print("[spawn]", " ".join(cmd))
125
+ PROC = subprocess.Popen(cmd, env=env,
126
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
127
+ text=True, bufsize=1)
128
+
129
+ for _ in range(600): # up to ~5 min for first cold load
130
+ if PROC.poll() is not None:
131
+ raise RuntimeError("llama-server died:\n" + (PROC.stdout.read() or ""))
132
+ try:
133
+ if requests.get(BASE + "/health", timeout=2).status_code == 200:
134
+ print("[llama-server] ready")
135
+ return
136
  except Exception:
137
+ pass
138
+ time.sleep(0.5)
139
+ raise RuntimeError("llama-server did not become healthy in time")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
140
 
141
+ @atexit.register
142
+ def _stop():
143
+ if PROC and PROC.poll() is None:
144
+ PROC.send_signal(signal.SIGTERM)
145
+
146
+ start_server()
147
+
148
+ # ----------------------------------------------------------------------------
149
+ # 4. INFERENCE (streaming, OpenAI schema)
150
+ # ----------------------------------------------------------------------------
151
+ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
152
+ message = (message or "").strip()
153
+ if not message:
154
+ yield ""
155
+ return
156
+
157
+ msgs = [{"role": "system", "content": system_prompt}] if system_prompt else []
158
+ for h in (history or []): # gradio "messages" format
159
+ if isinstance(h, dict) and h.get("content"):
160
+ msgs.append({"role": h["role"], "content": h["content"]})
161
+ msgs.append({"role": "user", "content": message})
162
+
163
+ payload = {"messages": msgs, "stream": True,
164
+ "max_tokens": int(max_tokens),
165
+ "temperature": float(temperature),
166
+ "top_p": float(top_p)}
167
+
168
+ out = ""
169
+ with requests.post(BASE + "/v1/chat/completions", json=payload,
170
+ stream=True, timeout=900) as r:
171
+ r.raise_for_status()
172
+ for line in r.iter_lines(decode_unicode=True):
173
+ if not line or not line.startswith("data: "):
174
+ continue
175
+ chunk = line[6:]
176
+ if chunk.strip() == "[DONE]":
177
+ break
178
+ try:
179
+ delta = json.loads(chunk)["choices"][0]["delta"].get("content")
180
+ except Exception:
181
+ continue
182
+ if delta:
183
+ out += delta
184
+ yield out
185
+
186
+ # ----------------------------------------------------------------------------
187
+ # 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
188
+ # ----------------------------------------------------------------------------
189
+ with gr.Blocks(title="GGUF Chat (llama.cpp prebuilt)") as demo:
190
+ gr.Markdown(f"# πŸ¦™ GGUF Chat β€” `{MODEL_REPO}`\nCPU inference via prebuilt `llama-server`.")
191
+
192
+ system = gr.Textbox(label="System prompt", value="You are a helpful assistant.")
 
 
 
 
 
 
 
 
 
 
 
 
193
  with gr.Row():
194
+ max_t = gr.Slider(16, 2048, value=512, step=16, label="Max new tokens")
195
+ temp = gr.Slider(0.0, 2.0, value=0.7, step=0.05, label="Temperature")
196
+ top_p = gr.Slider(0.1, 1.0, value=0.95, step=0.01, label="Top-p")
197
+
198
+ gr.ChatInterface(
199
+ fn=chat,
200
+ type="messages",
201
+ additional_inputs=[system, max_t, temp, top_p],
202
+ api_name="chat", # <- the external endpoint
203
+ examples=[["Explain GGUF quantization in two sentences."],
204
+ ["Write a haiku about CPUs."]],
205
+ )
206
+
207
+ demo.queue(max_size=20).launch(server_name="0.0.0.0",
208
+ server_port=int(os.getenv("GRADIO_PORT", "7860")),
209
+ ssr_mode=False,
210
+ show_api=True)
211
+ ```
212
+
213
+ ---
214
+
215
+ # 2. `requirements.txt`
216
+
217
+ ```
218
+ gradio==5.9.1
219
+ huggingface_hub>=0.26.0
220
+ requests>=2.32.0
221
+ ```
222
+
223
+ That's it β€” **no `llama-cpp-python`, no compiler, no `spaces`**. The engine is a downloaded ELF binary.
224
+
225
+ > If you *insist* on the Python binding instead of the HTTP server, the only no-compile way is a prebuilt wheel index:
226
+ > ```
227
+ > --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
228
+ > llama-cpp-python
229
+ > ```
230
+ > (wheel availability depends on your Python version β€” the subprocess approach above has zero such risk).
231
+
232
+ ---
233
+
234
+ # 3. Short doc β€” the important bits
235
+
236
+ ### A. How the original Edge-TTS code "avoids GPU"
237
+ | Thing | What it actually does |
238
+ |---|---|
239
+ | `import spaces` + `@spaces.GPU(duration=1)` | On **ZeroGPU** hardware, HF statically scans the file for a `@spaces.GPU` decorator at startup. No decorator β†’ the Space is rejected/ idles wrongly. The `dummy_gpu_trigger` is a **decoy** so the container boots. |
240
+ | `dummy_btn = gr.Button(visible=False)` | Keeps the decorated fn wired into the graph (so the scanner sees it used) while hiding it from users. |
241
+ | Real work (`edge_tts`) is pure network I/O | Microsoft's cloud does the synthesis β†’ **0 GPU seconds consumed**, no CUDA, no torch. |
242
+ | `duration=1` | If ZeroGPU ever *does* allocate, it reserves 1 second β€” negligible quota. |
243
+
244
+ πŸ‘‰ **On a plain CPU Space you should delete `import spaces` and the dummy entirely** (that's what I did above). Keep the trick only if the Space hardware is literally ZeroGPU and you want it to boot while doing CPU/network work.
245
+
246
+ ### B. How it becomes an external API
247
+ `api_name="speech"` / `api_name="chat"` publishes a named route. Everything else is Gradio's built-in HTTP layer.
248
+
249
+ **Python client**
250
+ ```python
251
+ from gradio_client import Client
252
+ c = Client("USER/SPACE") # or "http://localhost:7860"
253
+ print(c.predict("Hello", [], "You are helpful.", 512, 0.7, 0.95, api_name="/chat"))
254
+ ```
255
+
256
+ **Raw REST (Gradio 5 = 2-step)**
257
+ ```bash
258
+ EV=$(curl -s -XPOST https://USER-SPACE.hf.space/gradio_api/call/chat \
259
+ -H 'Content-Type: application/json' \
260
+ -d '{"data":["Hello",[],"You are helpful.",512,0.7,0.95]}' | jq -r .event_id)
261
+ curl -N https://USER-SPACE.hf.space/gradio_api/call/chat/$EV # SSE stream
262
+ ```
263
+ * Live schema: **`https://<space>.hf.space/?view=api`**
264
+ * Private Space β†’ header `Authorization: Bearer hf_xxx`.
265
+ * `show_api=True` + `demo.queue()` are required for stable concurrent API use.
266
+ * Generator functions stream over the API too (SSE), same as in the UI.
267
+
268
+ ### C. llama.cpp specifics in this app
269
+ * Binary: `ggml-org/llama.cpp` GitHub **Releases β†’ `llama-<tag>-bin-ubuntu-x64.zip`** (CPU, glibc). Pin it: `LLAMA_TAG=b4667` β€” `latest` can break asset names.
270
+ * Bundled `libllama.so`/`libggml*.so` sit next to the binary β†’ **`LD_LIBRARY_PATH` must include that dir** or you get "cannot open shared object file".
271
+ * `llama-server` speaks **OpenAI v1** (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`) + `/health`, `/props`. Chat template is read from the GGUF metadata β€” no manual prompt formatting.
272
+ * It stays on `127.0.0.1` β€” only Gradio is exposed. To expose it directly, mount it or proxy it (HF Spaces only routes **one** port, 7860).
273
+
274
+ ### D. Resources / gotchas for a ~1.6 GB Q4_K_M
275
+ * RAM β‰ˆ model (1.56 GB) + KV-cache. KV at 4096 ctx for a 2B β‰ˆ 0.3–0.6 GB β†’ fits free CPU Spaces (16 GB) easily; lower `CTX` if tight.
276
+ * Speed on free 2-vCPU: roughly **5–15 tok/s** for a 1–2B Q4. Set `THREADS` = physical cores; more threads than cores *hurts*.
277
+ * First boot downloads ~50 MB binary + model β†’ use `WORK_DIR=/data` on persistent storage, else it re-downloads every cold start.
278
+ * `-ngl 0` is mandatory for the CPU build; `-ngl 99` only works if you swap `LLAMA_ASSET` to a CUDA asset **and** run on GPU hardware (then you also need the matching cudart zip).
279
+ * `MODEL_REPO="openbmb/MiniCPM5-2B-GGUF"` β€” I can't confirm that repo exists (post-cutoff for me). If it 404s the code auto-falls back; just set `MODEL_REPO`/`MODEL_FILE` to whatever you see on the Hub.