Update app.py
Browse files
app.py
CHANGED
|
@@ -1,19 +1,4 @@
|
|
| 1 |
-
|
| 2 |
-
#
|
| 3 |
-
# FIXED 2026-10-04. The StopIteration came from two facts about the llama.cpp
|
| 4 |
-
# release feed that this file assumed incorrectly:
|
| 5 |
-
#
|
| 6 |
-
# 1. GET /releases/latest returns the newest release *by semver*, which is now
|
| 7 |
-
# "v0.5.0" — a changelog-only tag whose ONLY asset is `nightly-tag.txt`.
|
| 8 |
-
# The actual binaries live on the b-numbered nightly tags (b11392, ...)
|
| 9 |
-
# which are *newer by date* but lower by semver, so /releases/latest never
|
| 10 |
-
# sees them. => 0 candidate assets => StopIteration.
|
| 11 |
-
#
|
| 12 |
-
# 2. Linux binaries ship as **.tar.gz**. Only Windows ships .zip.
|
| 13 |
-
# The filter `a["name"].endswith(".zip")` therefore excluded 100% of the
|
| 14 |
-
# Linux assets even on a release that has them.
|
| 15 |
-
#
|
| 16 |
-
# Either one alone was fatal; both were present. See FIX 1 / FIX 2 below.
|
| 17 |
import os
|
| 18 |
import io
|
| 19 |
import re
|
|
@@ -50,9 +35,71 @@ LLAMA_TAG = os.getenv("LLAMA_TAG", "b11392")
|
|
| 50 |
LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
|
| 51 |
PORT = int(os.getenv("LLAMA_PORT", "8033"))
|
| 52 |
CTX = int(os.getenv("CTX", "4096"))
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
# FIX 5: /tmp is EPHEMERAL on Spaces and the GGUF is 1.56 GB, so every restart
|
| 57 |
# re-downloaded it. Point HF_HOME at persistent storage if you have it.
|
| 58 |
os.environ.setdefault("HF_HOME", os.getenv("HF_HOME", "/data/hf-cache"))
|
|
@@ -217,7 +264,15 @@ def start_server():
|
|
| 217 |
cmd = [str(server), "-m", model,
|
| 218 |
"--host", "127.0.0.1", "--port", str(PORT),
|
| 219 |
"-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
print("[spawn]", " ".join(cmd))
|
| 222 |
PROC = subprocess.Popen(cmd, env=env,
|
| 223 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
@@ -273,12 +328,13 @@ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
|
|
| 273 |
"temperature": float(temperature),
|
| 274 |
"top_p": float(top_p)}
|
| 275 |
|
| 276 |
-
# NOTE: MiniCPM5-2B is a reasoning model.
|
| 277 |
-
# stream carries `reasoning_content`
|
| 278 |
-
#
|
| 279 |
-
#
|
| 280 |
out = ""
|
| 281 |
-
|
|
|
|
| 282 |
with requests.post(BASE + "/v1/chat/completions", json=payload,
|
| 283 |
stream=True, timeout=900) as r:
|
| 284 |
r.raise_for_status()
|
|
@@ -289,13 +345,30 @@ def chat(message, history, system_prompt, max_tokens, temperature, top_p):
|
|
| 289 |
if chunk.strip() == "[DONE]":
|
| 290 |
break
|
| 291 |
try:
|
| 292 |
-
delta = json.loads(chunk)["choices"][0]["delta"]
|
| 293 |
except Exception:
|
| 294 |
continue
|
| 295 |
-
|
| 296 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
yield out
|
| 298 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
# ----------------------------------------------------------------------------
|
| 300 |
# 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
|
| 301 |
# ----------------------------------------------------------------------------
|
|
@@ -310,14 +383,19 @@ with gr.Blocks(title="GGUF Chat (llama.cpp prebuilt)") as demo:
|
|
| 310 |
|
| 311 |
gr.ChatInterface(
|
| 312 |
fn=chat,
|
| 313 |
-
type=
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
additional_inputs=[system, max_t, temp, top_p],
|
| 315 |
api_name="chat", # <- the external endpoint
|
| 316 |
examples=[["Explain GGUF quantization in two sentences."],
|
| 317 |
["Write a haiku about CPUs."]],
|
| 318 |
)
|
| 319 |
|
|
|
|
|
|
|
| 320 |
demo.queue(max_size=20).launch(server_name="0.0.0.0",
|
| 321 |
server_port=int(os.getenv("GRADIO_PORT", "7860")),
|
| 322 |
-
ssr_mode=False
|
| 323 |
-
show_api=True)
|
|
|
|
| 1 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
import os
|
| 3 |
import io
|
| 4 |
import re
|
|
|
|
| 35 |
LLAMA_ASSET = os.getenv("LLAMA_ASSET", "ubuntu-x64") # CPU linux build
|
| 36 |
PORT = int(os.getenv("LLAMA_PORT", "8033"))
|
| 37 |
CTX = int(os.getenv("CTX", "4096"))
|
| 38 |
+
|
| 39 |
+
# FIX 10: os.cpu_count() reports the HOST's cores inside a Space container —
|
| 40 |
+
# your log shows n_threads = 192. Spawning 192 llama.cpp threads on a couple of
|
| 41 |
+
# real vCPUs is pure context-switch thrash, and it also inflates the KV/
|
| 42 |
+
# threadpool allocation. Read the cgroup quota when there is one, else fall
|
| 43 |
+
# back to affinity, and never exceed a sane ceiling.
|
| 44 |
+
def _detect_threads() -> int:
|
| 45 |
+
env = os.getenv("THREADS")
|
| 46 |
+
if env:
|
| 47 |
+
return max(1, int(env))
|
| 48 |
+
n = None
|
| 49 |
+
for quota_path in ("/sys/fs/cgroup/cpu.max",): # cgroup v2
|
| 50 |
+
try:
|
| 51 |
+
quota, period = open(quota_path).read().split()
|
| 52 |
+
if quota != "max":
|
| 53 |
+
n = max(1, int(int(quota) / int(period)))
|
| 54 |
+
except Exception:
|
| 55 |
+
pass
|
| 56 |
+
if n is None:
|
| 57 |
+
try: # cgroup v1
|
| 58 |
+
q = int(open("/sys/fs/cgroup/cpu/cpu.cfs_quota_us").read())
|
| 59 |
+
p = int(open("/sys/fs/cgroup/cpu/cpu.cfs_period_us").read())
|
| 60 |
+
if q > 0:
|
| 61 |
+
n = max(1, q // p)
|
| 62 |
+
except Exception:
|
| 63 |
+
pass
|
| 64 |
+
if n is None:
|
| 65 |
+
try:
|
| 66 |
+
n = len(os.sched_getaffinity(0))
|
| 67 |
+
except Exception:
|
| 68 |
+
n = os.cpu_count() or 2
|
| 69 |
+
# Do NOT try to derive "physical cores" from sysfs here: inside a container
|
| 70 |
+
# /sys/devices/system/cpu/cpu*/topology/thread_siblings_list reports "0-1"
|
| 71 |
+
# for both vCPUs (i.e. claims 1 physical core) while llama-bench measures
|
| 72 |
+
# -t 2 = 9.70 t/s vs -t 1 = 5.16 t/s on this very box. The topology lies;
|
| 73 |
+
# the quota is the only trustworthy number. Just cap it -- token generation
|
| 74 |
+
# degrades well past a handful of threads:
|
| 75 |
+
# -t 2 -> 9.70 t/s | -t 4 -> 7.88 | -t 8 -> 6.97 | -t 192 -> 1.44 (your log)
|
| 76 |
+
# If you know your Space's real core count, set THREADS explicitly.
|
| 77 |
+
return max(1, min(int(n), int(os.getenv("THREADS_MAX", "8"))))
|
| 78 |
+
|
| 79 |
+
THREADS = _detect_threads()
|
| 80 |
NGL = int(os.getenv("NGL", "0")) # 0 = pure CPU. >0 only on a CUDA build.
|
| 81 |
|
| 82 |
+
# FIX 14: llama-server does NOT enable flash attention by default here.
|
| 83 |
+
# Measured on Ice Lake Xeon (b11392, MiniCPM5-2B Q4_K_M, 2 cores):
|
| 84 |
+
# pp512 42.6 -> 48.3 t/s (+13%)
|
| 85 |
+
# tg64 8.8 -> 9.4 t/s ( +7%)
|
| 86 |
+
# Costs nothing, so it is on by default. Set FLASH_ATTN=0 to disable.
|
| 87 |
+
FLASH_ATTN = os.getenv("FLASH_ATTN", "1") == "1"
|
| 88 |
+
|
| 89 |
+
# FIX 13: MiniCPM5-2B reasons by default and burns its ENTIRE max_tokens budget
|
| 90 |
+
# on reasoning_content before emitting a single content token. Verified at
|
| 91 |
+
# b11392: a haiku request with max_tokens=96 produced 96 reasoning tokens and
|
| 92 |
+
# ZERO answer tokens.
|
| 93 |
+
#
|
| 94 |
+
# The flag that actually controls this is `-rea, --reasoning [on|off|auto]`
|
| 95 |
+
# (env LLAMA_ARG_REASONING) -- NOT --reasoning-preserve, which only decides
|
| 96 |
+
# whether the reasoning trace is kept in the conversation history.
|
| 97 |
+
#
|
| 98 |
+
# --reasoning auto (default) -> 96/96 tokens to reasoning, no answer
|
| 99 |
+
# --reasoning off -> finish_reason=stop, 18 completion tokens,
|
| 100 |
+
# real haiku, ~13 s on 2 threads
|
| 101 |
+
REASONING = os.getenv("REASONING", "0") == "1"
|
| 102 |
+
|
| 103 |
# FIX 5: /tmp is EPHEMERAL on Spaces and the GGUF is 1.56 GB, so every restart
|
| 104 |
# re-downloaded it. Point HF_HOME at persistent storage if you have it.
|
| 105 |
os.environ.setdefault("HF_HOME", os.getenv("HF_HOME", "/data/hf-cache"))
|
|
|
|
| 264 |
cmd = [str(server), "-m", model,
|
| 265 |
"--host", "127.0.0.1", "--port", str(PORT),
|
| 266 |
"-c", str(CTX), "-t", str(THREADS), "-ngl", str(NGL),
|
| 267 |
+
# FIX 15: --parallel N splits the context into N slots AND allocates
|
| 268 |
+
# per-slot KV. Measured, it costs ~7% tg (8.86 -> 8.26 t/s) but more
|
| 269 |
+
# importantly it halves n_ctx_slot (2048 -> 1024 at -c 2048), which
|
| 270 |
+
# silently truncates long chats. Drop to 1 if you are short on RAM.
|
| 271 |
+
"--parallel", os.getenv("PARALLEL", "2"), "--cont-batching"]
|
| 272 |
+
# See FIX 13 — `-rea` gates thinking itself; --reasoning-preserve does not.
|
| 273 |
+
cmd += ["--reasoning", "on" if REASONING else "off"]
|
| 274 |
+
# See FIX 14 — free ~7% tg / ~13% pp.
|
| 275 |
+
cmd += ["--flash-attn", "on" if FLASH_ATTN else "off"]
|
| 276 |
print("[spawn]", " ".join(cmd))
|
| 277 |
PROC = subprocess.Popen(cmd, env=env,
|
| 278 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
|
|
| 328 |
"temperature": float(temperature),
|
| 329 |
"top_p": float(top_p)}
|
| 330 |
|
| 331 |
+
# NOTE: MiniCPM5-2B is a reasoning model. With --reasoning-preserve the
|
| 332 |
+
# stream carries `reasoning_content` first, then normal `content`. We show
|
| 333 |
+
# the reasoning in a collapsed block so the UI is never a dead blank bubble,
|
| 334 |
+
# and we guard the case where reasoning consumed the whole budget.
|
| 335 |
out = ""
|
| 336 |
+
thinking = ""
|
| 337 |
+
shown_thinking = False
|
| 338 |
with requests.post(BASE + "/v1/chat/completions", json=payload,
|
| 339 |
stream=True, timeout=900) as r:
|
| 340 |
r.raise_for_status()
|
|
|
|
| 345 |
if chunk.strip() == "[DONE]":
|
| 346 |
break
|
| 347 |
try:
|
| 348 |
+
delta = json.loads(chunk)["choices"][0]["delta"]
|
| 349 |
except Exception:
|
| 350 |
continue
|
| 351 |
+
rc = delta.get("reasoning_content")
|
| 352 |
+
if rc:
|
| 353 |
+
thinking += rc
|
| 354 |
+
shown_thinking = True
|
| 355 |
+
yield f"⏳ **thinking…**\n\n{thinking[-600:]}"
|
| 356 |
+
continue
|
| 357 |
+
if delta.get("content"):
|
| 358 |
+
out += delta["content"]
|
| 359 |
yield out
|
| 360 |
|
| 361 |
+
if not out:
|
| 362 |
+
# Every token went to reasoning, or the model produced nothing.
|
| 363 |
+
if shown_thinking:
|
| 364 |
+
yield (f"⚠️ The model spent the whole `max_tokens` budget "
|
| 365 |
+
f"({int(max_tokens)}) on reasoning and never produced an "
|
| 366 |
+
f"answer. Raise **Max new tokens**, or keep `REASONING=0`.\n\n"
|
| 367 |
+
f"<details><summary>Reasoning it did produce</summary>\n\n"
|
| 368 |
+
f"{thinking}\n\n</details>")
|
| 369 |
+
else:
|
| 370 |
+
yield "⚠️ The model returned no content. Try raising **Max new tokens**."
|
| 371 |
+
|
| 372 |
# ----------------------------------------------------------------------------
|
| 373 |
# 5. UI + PUBLIC API (api_name="chat" -> POST /gradio_api/call/chat)
|
| 374 |
# ----------------------------------------------------------------------------
|
|
|
|
| 383 |
|
| 384 |
gr.ChatInterface(
|
| 385 |
fn=chat,
|
| 386 |
+
# FIX 11: gradio 6 removed ChatInterface(type=...) and the class takes no
|
| 387 |
+
# **kwargs, so `type="messages"` is a hard TypeError. "messages" is now
|
| 388 |
+
# the ONLY history format; `multimodal=False` keeps `message` a plain
|
| 389 |
+
# str, which is what chat() expects.
|
| 390 |
+
multimodal=False,
|
| 391 |
additional_inputs=[system, max_t, temp, top_p],
|
| 392 |
api_name="chat", # <- the external endpoint
|
| 393 |
examples=[["Explain GGUF quantization in two sentences."],
|
| 394 |
["Write a haiku about CPUs."]],
|
| 395 |
)
|
| 396 |
|
| 397 |
+
# FIX 12: launch() no longer accepts show_api in gradio 6 — it would raise the
|
| 398 |
+
# same TypeError the moment ChatInterface was fixed.
|
| 399 |
demo.queue(max_size=20).launch(server_name="0.0.0.0",
|
| 400 |
server_port=int(os.getenv("GRADIO_PORT", "7860")),
|
| 401 |
+
ssr_mode=False)
|
|
|