Unmid commited on
Commit
780a801
·
verified ·
1 Parent(s): c8a08d8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +160 -40
app.py CHANGED
@@ -1,17 +1,32 @@
1
  # app.py — GGUF chat Space (prebuilt llama.cpp server from GitHub, NO local compile)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  import os
3
  import io
4
  import re
5
  import json
6
  import time
7
  import stat
 
8
  import zipfile
9
  import atexit
10
  import shutil
11
  import signal
12
  import socket
13
  import subprocess
14
- import urllib.request
15
  from pathlib import Path
16
 
17
  import requests
@@ -21,86 +36,175 @@ from huggingface_hub import hf_hub_download, list_repo_files
21
  # ----------------------------------------------------------------------------
22
  # 0. CONFIG (everything overridable with Space "Variables and secrets")
23
  # ----------------------------------------------------------------------------
24
- MODEL_REPO = os.getenv("MODEL_REPO", "openbmb/MiniCPM5-2B-GGUF") # <- verify it exists
25
  MODEL_FILE = os.getenv("MODEL_FILE", "") # "" = auto-pick Q4_K_M
26
  QUANT_HINT = os.getenv("QUANT_HINT", "Q4_K_M")
27
- FALLBACK_REPO= os.getenv("FALLBACK_REPO","bartowski/MiniCPM3-4B-GGUF")
 
 
28
 
29
- LLAMA_TAG = os.getenv("LLAMA_TAG", "latest") # pin e.g. "b4667" for reproducibility
 
 
 
30
  LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
31
  PORT = int(os.getenv("LLAMA_PORT", "8033"))
32
  CTX = int(os.getenv("CTX", "4096"))
33
  THREADS = int(os.getenv("THREADS", str(os.cpu_count() or 2)))
34
  NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
35
 
 
 
 
 
36
  WORK = Path(os.getenv("WORK_DIR", "/tmp/llamacpp"))
37
  WORK.mkdir(parents=True, exist_ok=True)
38
 
 
 
 
 
 
39
  # ----------------------------------------------------------------------------
40
  # 1. GET A PREBUILT llama.cpp BINARY FROM GITHUB RELEASES (no cmake, no gcc)
41
  # ----------------------------------------------------------------------------
42
- def _gh_release(tag: str) -> dict:
43
- url = ("https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
44
- if tag == "latest" else
45
- f"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{tag}")
46
- req = urllib.request.Request(url, headers={"User-Agent": "hf-space"})
47
- tok = os.getenv("GITHUB_TOKEN")
48
- if tok:
49
- req.add_header("Authorization", f"Bearer {tok}")
50
- with urllib.request.urlopen(req, timeout=60) as r:
51
- return json.load(r)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
52
 
53
  def ensure_llama_server() -> Path:
54
- cached = next(iter(WORK.glob("**/llama-server")), None)
55
  if cached:
 
 
56
  return cached
57
 
58
- rel = _gh_release(LLAMA_TAG)
59
- bad = ("cuda", "hip", "vulkan", "sycl", "musa", "arm64", "macos", "win")
60
- asset = next(
61
- a for a in rel["assets"]
62
- if a["name"].endswith(".zip")
63
- and LLAMA_ASSET in a["name"]
64
- and not any(b in a["name"].lower() for b in bad)
65
- )
66
- print(f"[llama.cpp] {rel['tag_name']} -> {asset['name']}")
67
 
68
- zpath = WORK / asset["name"]
69
- urllib.request.urlretrieve(asset["browser_download_url"], zpath)
70
- with zipfile.ZipFile(zpath) as z:
71
- z.extractall(WORK)
72
- zpath.unlink(missing_ok=True)
 
 
 
 
 
 
 
 
 
 
 
73
 
74
- server = next(WORK.glob("**/llama-server"))
75
- for f in server.parent.iterdir(): # binaries + bundled .so
76
- f.chmod(f.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  return server
78
 
79
  # ----------------------------------------------------------------------------
80
  # 2. GET THE GGUF FROM THE HUB
81
  # ----------------------------------------------------------------------------
82
  def ensure_model() -> str:
83
- for repo in (MODEL_REPO, FALLBACK_REPO):
 
 
84
  try:
85
  name = MODEL_FILE
86
  if not name:
87
  files = [f for f in list_repo_files(repo) if f.endswith(".gguf")]
88
- cand = [f for f in files if QUANT_HINT.lower() in f.lower()] or files
 
 
89
  # skip multi-part splits for simplicity
90
- cand = [f for f in cand if "-00002-of-" not in f]
91
- name = sorted(cand, key=len)[0]
92
  print(f"[model] {repo}/{name}")
93
  return hf_hub_download(repo_id=repo, filename=name,
94
  local_dir=str(WORK / "models"))
95
  except Exception as e:
 
96
  print(f"[model] {repo} failed: {e}")
97
- raise RuntimeError("No GGUF could be downloaded — set MODEL_REPO / MODEL_FILE.")
 
98
 
99
  # ----------------------------------------------------------------------------
100
  # 3. BOOT llama-server AS A SUBPROCESS (OpenAI-compatible endpoint on 127.0.0.1)
101
  # ----------------------------------------------------------------------------
102
  BASE = f"http://127.0.0.1:{PORT}"
103
  PROC = None
 
104
 
105
  def start_server():
106
  global PROC
@@ -119,9 +223,20 @@ def start_server():
119
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
120
  text=True, bufsize=1)
121
 
 
 
 
 
 
 
 
 
 
 
 
122
  for _ in range(600): # up to ~5 min for first cold load
123
  if PROC.poll() is not None:
124
- raise RuntimeError("llama-server died:\n" + (PROC.stdout.read() or ""))
125
  try:
126
  if requests.get(BASE + "/health", timeout=2).status_code == 200:
127
  print("[llama-server] ready")
@@ -129,7 +244,7 @@ def start_server():
129
  except Exception:
130
  pass
131
  time.sleep(0.5)
132
- raise RuntimeError("llama-server did not become healthy in time")
133
 
134
  @atexit.register
135
  def _stop():
@@ -158,7 +273,12 @@ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
158
  "temperature": float(temperature),
159
  "top_p": float(top_p)}
160
 
 
 
 
 
161
  out = ""
 
162
  with requests.post(BASE + "/v1/chat/completions", json=payload,
163
  stream=True, timeout=900) as r:
164
  r.raise_for_status()
@@ -200,4 +320,4 @@ with gr.Blocks(title="GGUF Chat (llama.cpp prebuilt)") as demo:
200
  demo.queue(max_size=20).launch(server_name="0.0.0.0",
201
  server_port=int(os.getenv("GRADIO_PORT", "7860")),
202
  ssr_mode=False,
203
- show_api=True)
 
1
  # app.py — GGUF chat Space (prebuilt llama.cpp server from GitHub, NO local compile)
2
+ #
3
+ # FIXED 2026-10-04. The StopIteration came from two facts about the llama.cpp
4
+ # release feed that this file assumed incorrectly:
5
+ #
6
+ # 1. GET /releases/latest returns the newest release *by semver*, which is now
7
+ # "v0.5.0" — a changelog-only tag whose ONLY asset is `nightly-tag.txt`.
8
+ # The actual binaries live on the b-numbered nightly tags (b11392, ...)
9
+ # which are *newer by date* but lower by semver, so /releases/latest never
10
+ # sees them. => 0 candidate assets => StopIteration.
11
+ #
12
+ # 2. Linux binaries ship as **.tar.gz**. Only Windows ships .zip.
13
+ # The filter `a["name"].endswith(".zip")` therefore excluded 100% of the
14
+ # Linux assets even on a release that has them.
15
+ #
16
+ # Either one alone was fatal; both were present. See FIX 1 / FIX 2 below.
17
  import os
18
  import io
19
  import re
20
  import json
21
  import time
22
  import stat
23
+ import tarfile
24
  import zipfile
25
  import atexit
26
  import shutil
27
  import signal
28
  import socket
29
  import subprocess
 
30
  from pathlib import Path
31
 
32
  import requests
 
36
  # ----------------------------------------------------------------------------
37
  # 0. CONFIG (everything overridable with Space "Variables and secrets")
38
  # ----------------------------------------------------------------------------
39
+ MODEL_REPO = os.getenv("MODEL_REPO", "openbmb/MiniCPM5-2B-GGUF") # verified public, not gated
40
  MODEL_FILE = os.getenv("MODEL_FILE", "") # "" = auto-pick Q4_K_M
41
  QUANT_HINT = os.getenv("QUANT_HINT", "Q4_K_M")
42
+ # FIX 4: the old fallback "bartowski/MiniCPM3-4B-GGUF" answers HTTP 401
43
+ # (gated or renamed), so the fallback chain was already dead.
44
+ FALLBACK_REPO= os.getenv("FALLBACK_REPO","")
45
 
46
+ # FIX 3: pin a build we have verified boots + serves. Set LLAMA_TAG=latest to
47
+ # track the newest nightly. A pin also stops a future asset rename from
48
+ # silently breaking the Space again.
49
+ LLAMA_TAG = os.getenv("LLAMA_TAG", "b11392")
50
  LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
51
  PORT = int(os.getenv("LLAMA_PORT", "8033"))
52
  CTX = int(os.getenv("CTX", "4096"))
53
  THREADS = int(os.getenv("THREADS", str(os.cpu_count() or 2)))
54
  NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
55
 
56
+ # FIX 5: /tmp is EPHEMERAL on Spaces and the GGUF is 1.56 GB, so every restart
57
+ # re-downloaded it. Point HF_HOME at persistent storage if you have it.
58
+ os.environ.setdefault("HF_HOME", os.getenv("HF_HOME", "/data/hf-cache"))
59
+
60
  WORK = Path(os.getenv("WORK_DIR", "/tmp/llamacpp"))
61
  WORK.mkdir(parents=True, exist_ok=True)
62
 
63
+ GH_HEADERS = {"User-Agent": "hf-space"}
64
+ _tok = os.getenv("GITHUB_TOKEN")
65
+ if _tok:
66
+ GH_HEADERS["Authorization"] = f"Bearer {_tok}"
67
+
68
  # ----------------------------------------------------------------------------
69
  # 1. GET A PREBUILT llama.cpp BINARY FROM GITHUB RELEASES (no cmake, no gcc)
70
  # ----------------------------------------------------------------------------
71
+ def _gh(url: str) -> list | dict:
72
+ r = requests.get(url, headers=GH_HEADERS, timeout=60)
73
+ r.raise_for_status()
74
+ return r.json()
75
+
76
+ def _has_linux_asset(rel: dict, needle: str) -> bool:
77
+ return any(a["name"].endswith(".tar.gz") and needle in a["name"]
78
+ for a in rel.get("assets", []))
79
+
80
+ def _resolve_release(tag: str, needle: str) -> dict:
81
+ """Return a release object that actually carries a Linux tarball.
82
+
83
+ FIX 1: never trust /releases/latest — it is semver-ordered and currently
84
+ resolves to the binary-less changelog tag v0.5.0.
85
+ """
86
+ if tag and tag != "latest":
87
+ rel = _gh(f"https://api.github.com/repos/ggml-org/llama.cpp/releases/tags/{tag}")
88
+ if _has_linux_asset(rel, needle):
89
+ return rel
90
+ print(f"[llama.cpp] pinned tag {tag} has no '{needle}' tarball; falling back to scan")
91
+
92
+ # Walk the release list newest-first and take the first that ships binaries.
93
+ # Drafts/prereleases are skipped; the b-numbered nightlies are neither.
94
+ rels = _gh("https://api.github.com/repos/ggml-org/llama.cpp/releases?per_page=50")
95
+ for rel in rels:
96
+ if rel.get("draft") or rel.get("prerelease"):
97
+ continue
98
+ if _has_linux_asset(rel, needle):
99
+ return rel
100
+ raise RuntimeError(
101
+ f"No llama.cpp release ships a Linux tarball matching '{needle}'. "
102
+ "Check https://github.com/ggml-org/llama.cpp/releases for the asset names."
103
+ )
104
+
105
+ def _pick_asset(rel: dict, needle: str) -> dict:
106
+ """Pick exactly one Linux CPU tarball, or fail loudly.
107
+
108
+ FIX 2: accept .tar.gz (Linux) as well as .zip (Windows).
109
+ FIX 6: on failure, print the real asset list instead of a bare StopIteration
110
+ that gives you nothing to debug from.
111
+ """
112
+ bad = ("cuda", "hip", "vulkan", "sycl", "musa", "openvino", "rocm",
113
+ "arm64", "macos", "android", "win", "snapdragon", "s390x")
114
+ cands = [a for a in rel["assets"]
115
+ if (a["name"].endswith(".tar.gz") or a["name"].endswith(".zip"))
116
+ and needle in a["name"]
117
+ and not any(b in a["name"].lower() for b in bad)]
118
+ if not cands:
119
+ listing = "\n".join(f" - {a['name']}" for a in sorted(rel["assets"], key=lambda x: x["name"]))
120
+ raise RuntimeError(
121
+ f"No asset in {rel['tag_name']} matches needle='{needle}'.\n"
122
+ f"Available assets ({len(rel['assets'])}):\n{listing}"
123
+ )
124
+ cands.sort(key=lambda a: (len(a["name"]), a["name"])) # deterministic
125
+ return cands[0]
126
 
127
  def ensure_llama_server() -> Path:
128
+ cached = next((p for p in WORK.glob("**/llama-server") if p.is_file()), None)
129
  if cached:
130
+ cached.chmod(cached.stat().st_mode | stat.S_IEXEC)
131
+ print(f"[llama.cpp] reusing cached {cached}")
132
  return cached
133
 
134
+ rel = _resolve_release(LLAMA_TAG, LLAMA_ASSET)
135
+ asset = _pick_asset(rel, LLAMA_ASSET)
136
+ print(f"[llama.cpp] {rel['tag_name']} -> {asset['name']} "
137
+ f"({asset['size'] / 1e6:.1f} MB)")
 
 
 
 
 
138
 
139
+ # FIX 7: urlretrieve sends no auth header and gives no retry. requests does.
140
+ arch = WORK / asset["name"]
141
+ for attempt in range(3):
142
+ try:
143
+ with requests.get(asset["browser_download_url"], headers=GH_HEADERS,
144
+ stream=True, timeout=300) as r:
145
+ r.raise_for_status()
146
+ with open(arch, "wb") as fh:
147
+ for blk in r.iter_content(1 << 20):
148
+ fh.write(blk)
149
+ break
150
+ except Exception as e:
151
+ print(f"[llama.cpp] download attempt {attempt + 1}/3 failed: {e}")
152
+ if attempt == 2:
153
+ raise
154
+ time.sleep(3)
155
 
156
+ if arch.name.endswith(".zip"):
157
+ with zipfile.ZipFile(arch) as z:
158
+ z.extractall(WORK)
159
+ else:
160
+ with tarfile.open(arch, "r:*") as t:
161
+ t.extractall(WORK, filter="data") # safe extract (py3.12+); drop for 3.10
162
+ arch.unlink(missing_ok=True)
163
+
164
+ server = next((p for p in WORK.glob("**/llama-server") if p.is_file()), None)
165
+ if server is None:
166
+ raise RuntimeError(f"llama-server not found after extracting {asset['name']} into {WORK}")
167
+
168
+ # FIX 8: the old loop chmod'ed every entry in server.parent including
169
+ # directories. Only touch files.
170
+ for f in server.parent.iterdir():
171
+ if f.is_file():
172
+ f.chmod(f.stat().st_mode | stat.S_IEXEC | stat.S_IRUSR)
173
+ server.chmod(server.stat().st_mode | stat.S_IEXEC)
174
  return server
175
 
176
  # ----------------------------------------------------------------------------
177
  # 2. GET THE GGUF FROM THE HUB
178
  # ----------------------------------------------------------------------------
179
  def ensure_model() -> str:
180
+ repos = [r for r in (MODEL_REPO, FALLBACK_REPO) if r]
181
+ errors = []
182
+ for repo in repos:
183
  try:
184
  name = MODEL_FILE
185
  if not name:
186
  files = [f for f in list_repo_files(repo) if f.endswith(".gguf")]
187
+ if not files:
188
+ raise RuntimeError(f"{repo} contains no .gguf files")
189
+ cand = [f for f in files if QUANT_HINT.lower() in f.lower()] or files
190
  # skip multi-part splits for simplicity
191
+ cand = [f for f in cand if "-00001-of-" not in f and "-00002-of-" not in f]
192
+ name = sorted(cand, key=len)[0]
193
  print(f"[model] {repo}/{name}")
194
  return hf_hub_download(repo_id=repo, filename=name,
195
  local_dir=str(WORK / "models"))
196
  except Exception as e:
197
+ errors.append(f"{repo}: {e}")
198
  print(f"[model] {repo} failed: {e}")
199
+ raise RuntimeError("No GGUF could be downloaded — set MODEL_REPO / MODEL_FILE.\n "
200
+ + "\n ".join(errors))
201
 
202
  # ----------------------------------------------------------------------------
203
  # 3. BOOT llama-server AS A SUBPROCESS (OpenAI-compatible endpoint on 127.0.0.1)
204
  # ----------------------------------------------------------------------------
205
  BASE = f"http://127.0.0.1:{PORT}"
206
  PROC = None
207
+ LOG = [] # FIX 9: keep server output so a crash is actually reportable
208
 
209
  def start_server():
210
  global PROC
 
223
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
224
  text=True, bufsize=1)
225
 
226
+ import threading
227
+ def _drain():
228
+ for line in PROC.stdout:
229
+ LOG.append(line)
230
+ if len(LOG) > 400:
231
+ del LOG[:200]
232
+ print("[llama]", line, end="")
233
+ threading.Thread(target=_drain, daemon=True).start()
234
+
235
+ # FIX 9: the old `PROC.stdout.read()` after the process died would block
236
+ # forever (the pipe was still held by the drain-less reader) or return "".
237
  for _ in range(600): # up to ~5 min for first cold load
238
  if PROC.poll() is not None:
239
+ raise RuntimeError("llama-server died:\n" + "".join(LOG[-80:]))
240
  try:
241
  if requests.get(BASE + "/health", timeout=2).status_code == 200:
242
  print("[llama-server] ready")
 
244
  except Exception:
245
  pass
246
  time.sleep(0.5)
247
+ raise RuntimeError("llama-server did not become healthy in time:\n" + "".join(LOG[-80:]))
248
 
249
  @atexit.register
250
  def _stop():
 
273
  "temperature": float(temperature),
274
  "top_p": float(top_p)}
275
 
276
+ # NOTE: MiniCPM5-2B is a reasoning model. Verified against b11392 — the
277
+ # stream carries `reasoning_content` tokens first, then normal `content`.
278
+ # Reading only `content` is correct; the UI just looks frozen for a few
279
+ # seconds while it thinks. Surface that instead of showing a blank bubble.
280
  out = ""
281
+ yield "⏳ thinking…"
282
  with requests.post(BASE + "/v1/chat/completions", json=payload,
283
  stream=True, timeout=900) as r:
284
  r.raise_for_status()
 
320
  demo.queue(max_size=20).launch(server_name="0.0.0.0",
321
  server_port=int(os.getenv("GRADIO_PORT", "7860")),
322
  ssr_mode=False,
323
+ show_api=True)