Unmid commited on
Commit
6dc2ebd
·
verified ·
1 Parent(s): 780a801

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +107 -29
app.py CHANGED
@@ -1,19 +1,4 @@
1
- # app.py — GGUF chat Space (prebuilt llama.cpp server from GitHub, NO local compile)
2
- #
3
- # FIXED 2026-10-04. The StopIteration came from two facts about the llama.cpp
4
- # release feed that this file assumed incorrectly:
5
- #
6
- # 1. GET /releases/latest returns the newest release *by semver*, which is now
7
- # "v0.5.0" — a changelog-only tag whose ONLY asset is `nightly-tag.txt`.
8
- # The actual binaries live on the b-numbered nightly tags (b11392, ...)
9
- # which are *newer by date* but lower by semver, so /releases/latest never
10
- # sees them. => 0 candidate assets => StopIteration.
11
- #
12
- # 2. Linux binaries ship as **.tar.gz**. Only Windows ships .zip.
13
- # The filter `a["name"].endswith(".zip")` therefore excluded 100% of the
14
- # Linux assets even on a release that has them.
15
- #
16
- # Either one alone was fatal; both were present. See FIX 1 / FIX 2 below.
17
  import os
18
  import io
19
  import re
@@ -50,9 +35,71 @@ LLAMA_TAG = os.getenv("LLAMA_TAG", "b11392")
50
  LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
51
  PORT = int(os.getenv("LLAMA_PORT", "8033"))
52
  CTX = int(os.getenv("CTX", "4096"))
53
- THREADS = int(os.getenv("THREADS", str(os.cpu_count() or 2)))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
55
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  # FIX 5: /tmp is EPHEMERAL on Spaces and the GGUF is 1.56 GB, so every restart
57
  # re-downloaded it. Point HF_HOME at persistent storage if you have it.
58
  os.environ.setdefault("HF_HOME", os.getenv("HF_HOME", "/data/hf-cache"))
@@ -217,7 +264,15 @@ def start_server():
217
  cmd = [str(server), "-m", model,
218
  "--host", "127.0.0.1", "--port", str(PORT),
219
  "-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
220
- "--parallel", "2", "--cont-batching"]
 
 
 
 
 
 
 
 
221
  print("[spawn]", " ".join(cmd))
222
  PROC = subprocess.Popen(cmd, env=env,
223
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
@@ -273,12 +328,13 @@ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
273
  "temperature": float(temperature),
274
  "top_p": float(top_p)}
275
 
276
- # NOTE: MiniCPM5-2B is a reasoning model. Verified against b11392 — the
277
- # stream carries `reasoning_content` tokens first, then normal `content`.
278
- # Reading only `content` is correct; the UI just looks frozen for a few
279
- # seconds while it thinks. Surface that instead of showing a blank bubble.
280
  out = ""
281
- yield "⏳ thinking…"
 
282
  with requests.post(BASE + "/v1/chat/completions", json=payload,
283
  stream=True, timeout=900) as r:
284
  r.raise_for_status()
@@ -289,13 +345,30 @@ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
289
  if chunk.strip() == "[DONE]":
290
  break
291
  try:
292
- delta = json.loads(chunk)["choices"][0]["delta"].get("content")
293
  except Exception:
294
  continue
295
- if delta:
296
- out += delta
 
 
 
 
 
 
297
  yield out
298
 
 
 
 
 
 
 
 
 
 
 
 
299
  # ----------------------------------------------------------------------------
300
  # 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
301
  # ----------------------------------------------------------------------------
@@ -310,14 +383,19 @@ with gr.Blocks(title="GGUF Chat (llama.cpp prebuilt)") as demo:
310
 
311
  gr.ChatInterface(
312
  fn=chat,
313
- type="messages",
 
 
 
 
314
  additional_inputs=[system, max_t, temp, top_p],
315
  api_name="chat", # <- the external endpoint
316
  examples=[["Explain GGUF quantization in two sentences."],
317
  ["Write a haiku about CPUs."]],
318
  )
319
 
 
 
320
  demo.queue(max_size=20).launch(server_name="0.0.0.0",
321
  server_port=int(os.getenv("GRADIO_PORT", "7860")),
322
- ssr_mode=False,
323
- show_api=True)
 
1
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  import os
3
  import io
4
  import re
 
35
  LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
36
  PORT = int(os.getenv("LLAMA_PORT", "8033"))
37
  CTX = int(os.getenv("CTX", "4096"))
38
+
39
+ # FIX 10: os.cpu_count() reports the HOST's cores inside a Space container —
40
+ # your log shows n_threads = 192. Spawning 192 llama.cpp threads on a couple of
41
+ # real vCPUs is pure context-switch thrash, and it also inflates the KV/
42
+ # threadpool allocation. Read the cgroup quota when there is one, else fall
43
+ # back to affinity, and never exceed a sane ceiling.
44
+ def _detect_threads() -> int:
45
+ env = os.getenv("THREADS")
46
+ if env:
47
+ return max(1, int(env))
48
+ n = None
49
+ for quota_path in ("/sys/fs/cgroup/cpu.max",): # cgroup v2
50
+ try:
51
+ quota, period = open(quota_path).read().split()
52
+ if quota != "max":
53
+ n = max(1, int(int(quota) / int(period)))
54
+ except Exception:
55
+ pass
56
+ if n is None:
57
+ try: # cgroup v1
58
+ q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
59
+ p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
60
+ if q > 0:
61
+ n = max(1, q // p)
62
+ except Exception:
63
+ pass
64
+ if n is None:
65
+ try:
66
+ n = len(os.sched_getaffinity(0))
67
+ except Exception:
68
+ n = os.cpu_count() or 2
69
+ # Do NOT try to derive "physical cores" from sysfs here: inside a container
70
+ # /sys/devices/system/cpu/cpu*/topology/thread_siblings_list reports "0-1"
71
+ # for both vCPUs (i.e. claims 1 physical core) while llama-bench measures
72
+ # -t 2 = 9.70 t/s vs -t 1 = 5.16 t/s on this very box. The topology lies;
73
+ # the quota is the only trustworthy number. Just cap it -- token generation
74
+ # degrades well past a handful of threads:
75
+ # -t 2 -> 9.70 t/s | -t 4 -> 7.88 | -t 8 -> 6.97 | -t 192 -> 1.44 (your log)
76
+ # If you know your Space's real core count, set THREADS explicitly.
77
+ return max(1, min(int(n), int(os.getenv("THREADS_MAX", "8"))))
78
+
79
+ THREADS = _detect_threads()
80
  NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
81
 
82
+ # FIX 14: llama-server does NOT enable flash attention by default here.
83
+ # Measured on Ice Lake Xeon (b11392, MiniCPM5-2B Q4_K_M, 2 cores):
84
+ # pp512 42.6 -> 48.3 t/s (+13%)
85
+ # tg64 8.8 -> 9.4 t/s ( +7%)
86
+ # Costs nothing, so it is on by default. Set FLASH_ATTN=0 to disable.
87
+ FLASH_ATTN = os.getenv("FLASH_ATTN", "1") == "1"
88
+
89
+ # FIX 13: MiniCPM5-2B reasons by default and burns its ENTIRE max_tokens budget
90
+ # on reasoning_content before emitting a single content token. Verified at
91
+ # b11392: a haiku request with max_tokens=96 produced 96 reasoning tokens and
92
+ # ZERO answer tokens.
93
+ #
94
+ # The flag that actually controls this is `-rea, --reasoning [on|off|auto]`
95
+ # (env LLAMA_ARG_REASONING) -- NOT --reasoning-preserve, which only decides
96
+ # whether the reasoning trace is kept in the conversation history.
97
+ #
98
+ # --reasoning auto (default) -> 96/96 tokens to reasoning, no answer
99
+ # --reasoning off -> finish_reason=stop, 18 completion tokens,
100
+ # real haiku, ~13 s on 2 threads
101
+ REASONING = os.getenv("REASONING", "0") == "1"
102
+
103
  # FIX 5: /tmp is EPHEMERAL on Spaces and the GGUF is 1.56 GB, so every restart
104
  # re-downloaded it. Point HF_HOME at persistent storage if you have it.
105
  os.environ.setdefault("HF_HOME", os.getenv("HF_HOME", "/data/hf-cache"))
 
264
  cmd = [str(server), "-m", model,
265
  "--host", "127.0.0.1", "--port", str(PORT),
266
  "-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
267
+ # FIX 15: --parallel N splits the context into N slots AND allocates
268
+ # per-slot KV. Measured, it costs ~7% tg (8.86 -> 8.26 t/s) but more
269
+ # importantly it halves n_ctx_slot (2048 -> 1024 at -c 2048), which
270
+ # silently truncates long chats. Drop to 1 if you are short on RAM.
271
+ "--parallel", os.getenv("PARALLEL", "2"), "--cont-batching"]
272
+ # See FIX 13 — `-rea` gates thinking itself; --reasoning-preserve does not.
273
+ cmd += ["--reasoning", "on" if REASONING else "off"]
274
+ # See FIX 14 — free ~7% tg / ~13% pp.
275
+ cmd += ["--flash-attn", "on" if FLASH_ATTN else "off"]
276
  print("[spawn]", " ".join(cmd))
277
  PROC = subprocess.Popen(cmd, env=env,
278
  stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 
328
  "temperature": float(temperature),
329
  "top_p": float(top_p)}
330
 
331
+ # NOTE: MiniCPM5-2B is a reasoning model. With --reasoning-preserve the
332
+ # stream carries `reasoning_content` first, then normal `content`. We show
333
+ # the reasoning in a collapsed block so the UI is never a dead blank bubble,
334
+ # and we guard the case where reasoning consumed the whole budget.
335
  out = ""
336
+ thinking = ""
337
+ shown_thinking = False
338
  with requests.post(BASE + "/v1/chat/completions", json=payload,
339
  stream=True, timeout=900) as r:
340
  r.raise_for_status()
 
345
  if chunk.strip() == "[DONE]":
346
  break
347
  try:
348
+ delta = json.loads(chunk)["choices"][0]["delta"]
349
  except Exception:
350
  continue
351
+ rc = delta.get("reasoning_content")
352
+ if rc:
353
+ thinking += rc
354
+ shown_thinking = True
355
+ yield f"⏳ **thinking…**\n\n{thinking[-600:]}"
356
+ continue
357
+ if delta.get("content"):
358
+ out += delta["content"]
359
  yield out
360
 
361
+ if not out:
362
+ # Every token went to reasoning, or the model produced nothing.
363
+ if shown_thinking:
364
+ yield (f"⚠️ The model spent the whole `max_tokens` budget "
365
+ f"({int(max_tokens)}) on reasoning and never produced an "
366
+ f"answer. Raise **Max new tokens**, or keep `REASONING=0`.\n\n"
367
+ f"<details><summary>Reasoning it did produce</summary>\n\n"
368
+ f"{thinking}\n\n</details>")
369
+ else:
370
+ yield "⚠️ The model returned no content. Try raising **Max new tokens**."
371
+
372
  # ----------------------------------------------------------------------------
373
  # 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
374
  # ----------------------------------------------------------------------------
 
383
 
384
  gr.ChatInterface(
385
  fn=chat,
386
+ # FIX 11: gradio 6 removed ChatInterface(type=...) and the class takes no
387
+ # **kwargs, so `type="messages"` is a hard TypeError. "messages" is now
388
+ # the ONLY history format; `multimodal=False` keeps `message` a plain
389
+ # str, which is what chat() expects.
390
+ multimodal=False,
391
  additional_inputs=[system, max_t, temp, top_p],
392
  api_name="chat", # <- the external endpoint
393
  examples=[["Explain GGUF quantization in two sentences."],
394
  ["Write a haiku about CPUs."]],
395
  )
396
 
397
+ # FIX 12: launch() no longer accepts show_api in gradio 6 — it would raise the
398
+ # same TypeError the moment ChatInterface was fixed.
399
  demo.queue(max_size=20).launch(server_name="0.0.0.0",
400
  server_port=int(os.getenv("GRADIO_PORT", "7860")),
401
+ ssr_mode=False)