owenisas commited on
Commit
a8d9fe7
·
verified ·
1 Parent(s): a38bb98

Serve GGUF with persistent llama.cpp server

Browse files
Files changed (3) hide show
  1. README.md +5 -5
  2. __pycache__/app.cpython-314.pyc +0 -0
  3. app.py +139 -67
README.md CHANGED
@@ -28,7 +28,7 @@ license: mit
28
 
29
  First-Principle AI is a compact Gradio console for running and probing the
30
  `build-small-hackathon/phase-3-gguf` Q8 GGUF model through
31
- the official `llama.cpp` Ubuntu CLI release.
32
 
33
  The UI includes benchmark-style examples inspired by common LLM evaluation
34
  areas: math reasoning, commonsense, science QA, truthfulness, instruction
@@ -39,12 +39,12 @@ questions are original prompts, not copied benchmark items.
39
 
40
  - Model repo: `build-small-hackathon/phase-3-gguf`
41
  - Model file: `model-Q8_0.gguf`
42
- - Runtime: official `llama.cpp` `llama-cli`
43
  - Hardware target: ZeroGPU
44
  - Fallback behavior: visible runtime diagnostics instead of silent mock output
45
- - Model loading: runtime download/load through `llama-cli`
46
- - Default llama.cpp settings: `n_ctx=4096`, `n_batch=512`, `n_ubatch=128`,
47
- memory-mapped weights, and CPU fallback if CUDA offload is unavailable
48
 
49
  ZeroGPU is a Gradio dynamic GPU runtime primarily documented around PyTorch
50
  workloads. This app targets ZeroGPU as requested, but it runs the GGUF through
 
28
 
29
  First-Principle AI is a compact Gradio console for running and probing the
30
  `build-small-hackathon/phase-3-gguf` Q8 GGUF model through
31
+ the official `llama.cpp` Ubuntu `llama-server` release.
32
 
33
  The UI includes benchmark-style examples inspired by common LLM evaluation
34
  areas: math reasoning, commonsense, science QA, truthfulness, instruction
 
39
 
40
  - Model repo: `build-small-hackathon/phase-3-gguf`
41
  - Model file: `model-Q8_0.gguf`
42
+ - Runtime: official `llama.cpp` `llama-server`
43
  - Hardware target: ZeroGPU
44
  - Fallback behavior: visible runtime diagnostics instead of silent mock output
45
+ - Model loading: runtime download/load through a persistent `llama-server`
46
+ - Default llama.cpp settings: `n_ctx=2048`, `n_batch=256`, `n_ubatch=64`,
47
+ memory-mapped weights, no warmup, and CPU fallback if CUDA offload is unavailable
48
 
49
  ZeroGPU is a Gradio dynamic GPU runtime primarily documented around PyTorch
50
  workloads. This app targets ZeroGPU as requested, but it runs the GGUF through
__pycache__/app.cpython-314.pyc CHANGED
Binary files a/__pycache__/app.cpython-314.pyc and b/__pycache__/app.cpython-314.pyc differ
 
app.py CHANGED
@@ -8,6 +8,7 @@ import time
8
  import subprocess
9
  import tarfile
10
  import urllib.request
 
11
  from pathlib import Path
12
  from typing import Any
13
 
@@ -28,12 +29,12 @@ LLAMA_URL = os.getenv(
28
  "PHASE3_LLAMA_URL",
29
  f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_RELEASE}/llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz",
30
  )
31
- MAX_CONTEXT = int(os.getenv("PHASE3_MAX_CONTEXT", "4096"))
32
  MIN_RAM_GB = float(os.getenv("PHASE3_MIN_RAM_GB", "38"))
33
  DISABLE_MODEL = os.getenv("PHASE3_DISABLE_MODEL", "").lower() in {"1", "true", "yes"}
34
  USE_ZEROGPU_DECORATOR = os.getenv("PHASE3_USE_ZEROGPU", "").lower() in {"1", "true", "yes"}
35
- N_BATCH = int(os.getenv("PHASE3_N_BATCH", "512"))
36
- N_UBATCH = int(os.getenv("PHASE3_N_UBATCH", "128"))
37
  N_THREADS = int(os.getenv("PHASE3_THREADS", str(max(1, min(16, os.cpu_count() or 2)))))
38
  N_THREADS_BATCH = int(os.getenv("PHASE3_THREADS_BATCH", str(N_THREADS)))
39
  USE_MMAP = os.getenv("PHASE3_USE_MMAP", "1").lower() not in {"0", "false", "no"}
@@ -41,10 +42,15 @@ USE_MLOCK = os.getenv("PHASE3_USE_MLOCK", "").lower() in {"1", "true", "yes"}
41
  FLASH_ATTN = os.getenv("PHASE3_FLASH_ATTN", "").lower() in {"1", "true", "yes"}
42
  OFFLOAD_KQV = os.getenv("PHASE3_OFFLOAD_KQV", "1").lower() not in {"0", "false", "no"}
43
  INFER_TIMEOUT = int(os.getenv("PHASE3_INFER_TIMEOUT", "900"))
 
 
 
44
 
45
  MODEL_LOCK = threading.Lock()
46
  MODEL_PATH: Path | None = None
47
  LLAMA_CLI_PATH: Path | None = None
 
 
48
  MODEL_ERROR: str | None = None
49
  MODEL_SETTINGS: dict[str, Any] = {}
50
 
@@ -151,18 +157,24 @@ def _gpu_layers() -> int:
151
  return 0
152
 
153
 
154
- def _ensure_llama_cli() -> Path:
155
- global LLAMA_CLI_PATH
156
 
157
- if LLAMA_CLI_PATH is not None and LLAMA_CLI_PATH.exists():
158
  return LLAMA_CLI_PATH
 
 
159
 
160
  root = Path(os.getenv("PHASE3_LLAMA_DIR", "/tmp/phase3-llama.cpp"))
161
  release_dir = root / f"llama-{LLAMA_RELEASE}"
162
- cli = release_dir / "llama-cli"
163
- if cli.exists():
164
- LLAMA_CLI_PATH = cli
165
- return cli
 
 
 
 
166
 
167
  root.mkdir(parents=True, exist_ok=True)
168
  archive = root / f"llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz"
@@ -170,11 +182,14 @@ def _ensure_llama_cli() -> Path:
170
  urllib.request.urlretrieve(LLAMA_URL, archive)
171
  with tarfile.open(archive, "r:gz") as tar:
172
  tar.extractall(root)
173
- if not cli.exists():
174
- raise RuntimeError(f"llama-cli was not found after extracting {LLAMA_URL}")
175
- cli.chmod(0o755)
176
- LLAMA_CLI_PATH = cli
177
- return cli
 
 
 
178
 
179
 
180
  def _prepare_runtime() -> tuple[Path, Path]:
@@ -196,12 +211,12 @@ def _prepare_runtime() -> tuple[Path, Path]:
196
  raise RuntimeError(MODEL_ERROR)
197
 
198
  path = _find_model_path()
199
- cli = _ensure_llama_cli()
200
  MODEL_PATH = path
201
  n_gpu_layers = _gpu_layers()
202
  MODEL_SETTINGS = {
203
  "path": str(path),
204
- "llama_cli": str(cli),
205
  "n_ctx": MAX_CONTEXT,
206
  "n_batch": N_BATCH,
207
  "n_ubatch": N_UBATCH,
@@ -212,41 +227,50 @@ def _prepare_runtime() -> tuple[Path, Path]:
212
  "use_mlock": USE_MLOCK,
213
  "flash_attn": FLASH_ATTN,
214
  "offload_kqv": OFFLOAD_KQV,
 
215
  }
216
- return path, cli
217
 
218
 
219
- def _format_prompt(system_prompt: str, history: list[dict[str, str]], message: str) -> str:
220
- system = system_prompt.strip() or "You are a precise, direct model in a technical lab console."
221
- turns = [f"<|im_start|>system\n{system}<|im_end|>"]
222
- for item in history[-10:]:
223
- role = item.get("role", "user")
224
- content = item.get("content", "")
225
- if role in {"user", "assistant"} and content:
226
- turns.append(f"<|im_start|>{role}\n{content}<|im_end|>")
227
- turns.append(f"<|im_start|>user\n{message}<|im_end|>")
228
- turns.append("<|im_start|>assistant\n")
229
- return "\n".join(turns)
230
 
231
 
232
- @_gpu_decorator
233
- def _complete(
234
- prompt: str,
235
- max_tokens: int,
236
- temperature: float,
237
- top_p: float,
238
- repeat_penalty: float,
239
- ) -> tuple[str, dict[str, Any]]:
240
- model_path, llama_cli = _prepare_runtime()
241
- started = time.time()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
242
  cmd = [
243
- str(llama_cli),
244
  "-m",
245
  str(model_path),
246
- "-p",
247
- prompt,
248
- "-n",
249
- str(int(max_tokens)),
250
  "-c",
251
  str(MAX_CONTEXT),
252
  "-t",
@@ -255,13 +279,6 @@ def _complete(
255
  str(N_BATCH),
256
  "-ub",
257
  str(N_UBATCH),
258
- "--temp",
259
- str(float(temperature)),
260
- "--top-p",
261
- str(float(top_p)),
262
- "--repeat-penalty",
263
- str(float(repeat_penalty)),
264
- "--no-display-prompt",
265
  ]
266
  if _gpu_layers() != 0:
267
  cmd.extend(["-ngl", str(_gpu_layers())])
@@ -271,25 +288,79 @@ def _complete(
271
  cmd.append("--no-mmap")
272
  if FLASH_ATTN:
273
  cmd.append("-fa")
 
 
274
 
275
  env = os.environ.copy()
276
- binary_dir = str(llama_cli.parent)
277
  env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
278
- proc = subprocess.run(
 
 
 
 
279
  cmd,
280
  cwd=binary_dir,
281
  env=env,
 
 
282
  text=True,
283
- capture_output=True,
284
- timeout=INFER_TIMEOUT,
285
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
  elapsed = max(time.time() - started, 0.001)
287
- if proc.returncode != 0:
288
- stderr = proc.stderr.strip()
289
- stdout = proc.stdout.strip()
290
- detail = stderr or stdout or f"llama-cli exited with code {proc.returncode}"
291
- raise RuntimeError(detail[-4000:])
292
- text = proc.stdout.strip()
293
  text = text.split("<|im_end|>", 1)[0].strip()
294
  completion_tokens = max(1, len(text.split()))
295
  return text, {
@@ -308,7 +379,8 @@ def _status_markdown() -> str:
308
  model_state = "Ready" if MODEL_PATH is not None else ("Error" if MODEL_ERROR else "Ready to load on first prompt")
309
  available_text = f"{available_gb:.1f} GB" if available_gb is not None else "unknown"
310
  path_text = f"`{MODEL_PATH}`" if MODEL_PATH else "not resolved yet"
311
- cli_text = f"`{LLAMA_CLI_PATH}`" if LLAMA_CLI_PATH else f"`{LLAMA_RELEASE}` not extracted yet"
 
312
  settings = MODEL_SETTINGS or {
313
  "n_ctx": MAX_CONTEXT,
314
  "n_batch": N_BATCH,
@@ -335,11 +407,11 @@ def _status_markdown() -> str:
335
  | Available RAM | {available_text} |
336
  | CUDA devices | `{cuda_text}` |
337
  | Model path | {path_text} |
338
- | llama-cli | {cli_text} |
339
  | llama.cpp settings | `ctx={settings.get('n_ctx')}`, `batch={settings.get('n_batch')}`, `ubatch={settings.get('n_ubatch')}`, `threads={settings.get('n_threads')}`, `gpu_layers={settings.get('n_gpu_layers')}` |
340
- | Memory/options | `mmap={settings.get('use_mmap')}`, `mlock={settings.get('use_mlock')}`, `flash_attn={settings.get('flash_attn')}`, `offload_kqv={settings.get('offload_kqv')}` |
341
 
342
- The first prompt downloads the 31 GB Q8 GGUF and the llama.cpp binary if they are not cached. Generation runs through `llama-cli`.
343
  """
344
 
345
 
@@ -396,7 +468,7 @@ def respond(
396
  "Model load or inference failed.\n\n"
397
  f"{exc}\n\n"
398
  "The UI is live and the model artifact is published, but the runtime could not complete "
399
- "a llama.cpp CLI generation pass. Check the runtime status and Space logs before retrying."
400
  )
401
  meta = {"elapsed": 0.0, "completion_tokens": len(text.split()), "tokens_per_second": 0.0}
402
 
@@ -548,7 +620,7 @@ with gr.Blocks(title="First-Principle AI", fill_width=True) as demo:
548
  <p>A clean model-console interface for probing the Phase-3 Q8 GGUF with transparent runtime status.</p>
549
  <div class="phase-badge-row">
550
  <span class="phase-badge"><strong>Model</strong> build-small-hackathon/phase-3-gguf</span>
551
- <span class="phase-badge"><strong>Runtime</strong> llama.cpp CLI</span>
552
  <span class="phase-badge"><strong>Mode</strong> real GGUF inference</span>
553
  </div>
554
  </div>
 
8
  import subprocess
9
  import tarfile
10
  import urllib.request
11
+ import json
12
  from pathlib import Path
13
  from typing import Any
14
 
 
29
  "PHASE3_LLAMA_URL",
30
  f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_RELEASE}/llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz",
31
  )
32
+ MAX_CONTEXT = int(os.getenv("PHASE3_MAX_CONTEXT", "2048"))
33
  MIN_RAM_GB = float(os.getenv("PHASE3_MIN_RAM_GB", "38"))
34
  DISABLE_MODEL = os.getenv("PHASE3_DISABLE_MODEL", "").lower() in {"1", "true", "yes"}
35
  USE_ZEROGPU_DECORATOR = os.getenv("PHASE3_USE_ZEROGPU", "").lower() in {"1", "true", "yes"}
36
+ N_BATCH = int(os.getenv("PHASE3_N_BATCH", "256"))
37
+ N_UBATCH = int(os.getenv("PHASE3_N_UBATCH", "64"))
38
  N_THREADS = int(os.getenv("PHASE3_THREADS", str(max(1, min(16, os.cpu_count() or 2)))))
39
  N_THREADS_BATCH = int(os.getenv("PHASE3_THREADS_BATCH", str(N_THREADS)))
40
  USE_MMAP = os.getenv("PHASE3_USE_MMAP", "1").lower() not in {"0", "false", "no"}
 
42
  FLASH_ATTN = os.getenv("PHASE3_FLASH_ATTN", "").lower() in {"1", "true", "yes"}
43
  OFFLOAD_KQV = os.getenv("PHASE3_OFFLOAD_KQV", "1").lower() not in {"0", "false", "no"}
44
  INFER_TIMEOUT = int(os.getenv("PHASE3_INFER_TIMEOUT", "900"))
45
+ SERVER_HOST = "127.0.0.1"
46
+ SERVER_PORT = int(os.getenv("PHASE3_SERVER_PORT", "8088"))
47
+ NO_WARMUP = os.getenv("PHASE3_NO_WARMUP", "1").lower() not in {"0", "false", "no"}
48
 
49
  MODEL_LOCK = threading.Lock()
50
  MODEL_PATH: Path | None = None
51
  LLAMA_CLI_PATH: Path | None = None
52
+ LLAMA_SERVER_PATH: Path | None = None
53
+ LLAMA_SERVER_PROCESS: subprocess.Popen[str] | None = None
54
  MODEL_ERROR: str | None = None
55
  MODEL_SETTINGS: dict[str, Any] = {}
56
 
 
157
  return 0
158
 
159
 
160
+ def _ensure_llama_binary(name: str) -> Path:
161
+ global LLAMA_CLI_PATH, LLAMA_SERVER_PATH
162
 
163
+ if name == "llama-cli" and LLAMA_CLI_PATH is not None and LLAMA_CLI_PATH.exists():
164
  return LLAMA_CLI_PATH
165
+ if name == "llama-server" and LLAMA_SERVER_PATH is not None and LLAMA_SERVER_PATH.exists():
166
+ return LLAMA_SERVER_PATH
167
 
168
  root = Path(os.getenv("PHASE3_LLAMA_DIR", "/tmp/phase3-llama.cpp"))
169
  release_dir = root / f"llama-{LLAMA_RELEASE}"
170
+ binary = release_dir / name
171
+ if binary.exists():
172
+ binary.chmod(0o755)
173
+ if name == "llama-cli":
174
+ LLAMA_CLI_PATH = binary
175
+ if name == "llama-server":
176
+ LLAMA_SERVER_PATH = binary
177
+ return binary
178
 
179
  root.mkdir(parents=True, exist_ok=True)
180
  archive = root / f"llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz"
 
182
  urllib.request.urlretrieve(LLAMA_URL, archive)
183
  with tarfile.open(archive, "r:gz") as tar:
184
  tar.extractall(root)
185
+ if not binary.exists():
186
+ raise RuntimeError(f"{name} was not found after extracting {LLAMA_URL}")
187
+ binary.chmod(0o755)
188
+ if name == "llama-cli":
189
+ LLAMA_CLI_PATH = binary
190
+ if name == "llama-server":
191
+ LLAMA_SERVER_PATH = binary
192
+ return binary
193
 
194
 
195
  def _prepare_runtime() -> tuple[Path, Path]:
 
211
  raise RuntimeError(MODEL_ERROR)
212
 
213
  path = _find_model_path()
214
+ server = _ensure_llama_binary("llama-server")
215
  MODEL_PATH = path
216
  n_gpu_layers = _gpu_layers()
217
  MODEL_SETTINGS = {
218
  "path": str(path),
219
+ "llama_server": str(server),
220
  "n_ctx": MAX_CONTEXT,
221
  "n_batch": N_BATCH,
222
  "n_ubatch": N_UBATCH,
 
227
  "use_mlock": USE_MLOCK,
228
  "flash_attn": FLASH_ATTN,
229
  "offload_kqv": OFFLOAD_KQV,
230
+ "no_warmup": NO_WARMUP,
231
  }
232
+ return path, server
233
 
234
 
235
+ def _server_log_path() -> Path:
236
+ return Path(os.getenv("PHASE3_SERVER_LOG", "/tmp/phase3-llama-server.log"))
 
 
 
 
 
 
 
 
 
237
 
238
 
239
+ def _tail_server_log(limit: int = 4000) -> str:
240
+ path = _server_log_path()
241
+ if not path.exists():
242
+ return ""
243
+ data = path.read_text(encoding="utf-8", errors="ignore")
244
+ return data[-limit:]
245
+
246
+
247
+ def _server_url(path: str) -> str:
248
+ return f"http://{SERVER_HOST}:{SERVER_PORT}{path}"
249
+
250
+
251
+ def _server_is_ready() -> bool:
252
+ try:
253
+ with urllib.request.urlopen(_server_url("/health"), timeout=5) as resp:
254
+ return 200 <= resp.status < 500
255
+ except Exception:
256
+ return False
257
+
258
+
259
+ def _start_server() -> None:
260
+ global LLAMA_SERVER_PROCESS
261
+
262
+ model_path, server = _prepare_runtime()
263
+ if LLAMA_SERVER_PROCESS is not None and LLAMA_SERVER_PROCESS.poll() is None and _server_is_ready():
264
+ return
265
+
266
  cmd = [
267
+ str(server),
268
  "-m",
269
  str(model_path),
270
+ "--host",
271
+ SERVER_HOST,
272
+ "--port",
273
+ str(SERVER_PORT),
274
  "-c",
275
  str(MAX_CONTEXT),
276
  "-t",
 
279
  str(N_BATCH),
280
  "-ub",
281
  str(N_UBATCH),
 
 
 
 
 
 
 
282
  ]
283
  if _gpu_layers() != 0:
284
  cmd.extend(["-ngl", str(_gpu_layers())])
 
288
  cmd.append("--no-mmap")
289
  if FLASH_ATTN:
290
  cmd.append("-fa")
291
+ if NO_WARMUP:
292
+ cmd.append("--no-warmup")
293
 
294
  env = os.environ.copy()
295
+ binary_dir = str(server.parent)
296
  env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
297
+ log_path = _server_log_path()
298
+ log_file = log_path.open("a", encoding="utf-8")
299
+ log_file.write(f"\n--- starting llama-server: {' '.join(cmd)} ---\n")
300
+ log_file.flush()
301
+ LLAMA_SERVER_PROCESS = subprocess.Popen(
302
  cmd,
303
  cwd=binary_dir,
304
  env=env,
305
+ stdout=log_file,
306
+ stderr=subprocess.STDOUT,
307
  text=True,
 
 
308
  )
309
+
310
+ deadline = time.time() + INFER_TIMEOUT
311
+ while time.time() < deadline:
312
+ if LLAMA_SERVER_PROCESS.poll() is not None:
313
+ raise RuntimeError(f"llama-server exited early.\n{_tail_server_log()}")
314
+ if _server_is_ready():
315
+ return
316
+ time.sleep(2)
317
+ raise RuntimeError(f"llama-server did not become ready within {INFER_TIMEOUT}s.\n{_tail_server_log()}")
318
+
319
+
320
+ def _format_prompt(system_prompt: str, history: list[dict[str, str]], message: str) -> str:
321
+ system = system_prompt.strip() or "You are a precise, direct model in a technical lab console."
322
+ turns = [f"<|im_start|>system\n{system}<|im_end|>"]
323
+ for item in history[-10:]:
324
+ role = item.get("role", "user")
325
+ content = item.get("content", "")
326
+ if role in {"user", "assistant"} and content:
327
+ turns.append(f"<|im_start|>{role}\n{content}<|im_end|>")
328
+ turns.append(f"<|im_start|>user\n{message}<|im_end|>")
329
+ turns.append("<|im_start|>assistant\n")
330
+ return "\n".join(turns)
331
+
332
+
333
+ @_gpu_decorator
334
+ def _complete(
335
+ prompt: str,
336
+ max_tokens: int,
337
+ temperature: float,
338
+ top_p: float,
339
+ repeat_penalty: float,
340
+ ) -> tuple[str, dict[str, Any]]:
341
+ started = time.time()
342
+ _start_server()
343
+ payload = {
344
+ "prompt": prompt,
345
+ "n_predict": int(max_tokens),
346
+ "temperature": float(temperature),
347
+ "top_p": float(top_p),
348
+ "repeat_penalty": float(repeat_penalty),
349
+ "stop": ["<|im_end|>", "<|endoftext|>"],
350
+ }
351
+ req = urllib.request.Request(
352
+ _server_url("/completion"),
353
+ data=json.dumps(payload).encode("utf-8"),
354
+ headers={"Content-Type": "application/json"},
355
+ method="POST",
356
+ )
357
+ try:
358
+ with urllib.request.urlopen(req, timeout=INFER_TIMEOUT) as resp:
359
+ output = json.loads(resp.read().decode("utf-8"))
360
+ except Exception as exc:
361
+ raise RuntimeError(f"llama-server completion failed: {exc}\n{_tail_server_log()}") from exc
362
  elapsed = max(time.time() - started, 0.001)
363
+ text = (output.get("content") or "").strip()
 
 
 
 
 
364
  text = text.split("<|im_end|>", 1)[0].strip()
365
  completion_tokens = max(1, len(text.split()))
366
  return text, {
 
379
  model_state = "Ready" if MODEL_PATH is not None else ("Error" if MODEL_ERROR else "Ready to load on first prompt")
380
  available_text = f"{available_gb:.1f} GB" if available_gb is not None else "unknown"
381
  path_text = f"`{MODEL_PATH}`" if MODEL_PATH else "not resolved yet"
382
+ server_text = f"`{LLAMA_SERVER_PATH}`" if LLAMA_SERVER_PATH else f"`{LLAMA_RELEASE}` not extracted yet"
383
+ server_state = "running" if LLAMA_SERVER_PROCESS is not None and LLAMA_SERVER_PROCESS.poll() is None else "not started"
384
  settings = MODEL_SETTINGS or {
385
  "n_ctx": MAX_CONTEXT,
386
  "n_batch": N_BATCH,
 
407
  | Available RAM | {available_text} |
408
  | CUDA devices | `{cuda_text}` |
409
  | Model path | {path_text} |
410
+ | llama-server | {server_text} ({server_state}) |
411
  | llama.cpp settings | `ctx={settings.get('n_ctx')}`, `batch={settings.get('n_batch')}`, `ubatch={settings.get('n_ubatch')}`, `threads={settings.get('n_threads')}`, `gpu_layers={settings.get('n_gpu_layers')}` |
412
+ | Memory/options | `mmap={settings.get('use_mmap')}`, `mlock={settings.get('use_mlock')}`, `flash_attn={settings.get('flash_attn')}`, `no_warmup={settings.get('no_warmup')}` |
413
 
414
+ The first prompt starts `llama-server` and loads the 31 GB Q8 GGUF if it is not already cached. Later prompts reuse the same llama.cpp server process.
415
  """
416
 
417
 
 
468
  "Model load or inference failed.\n\n"
469
  f"{exc}\n\n"
470
  "The UI is live and the model artifact is published, but the runtime could not complete "
471
+ "a llama.cpp server generation pass. Check the runtime status and Space logs before retrying."
472
  )
473
  meta = {"elapsed": 0.0, "completion_tokens": len(text.split()), "tokens_per_second": 0.0}
474
 
 
620
  <p>A clean model-console interface for probing the Phase-3 Q8 GGUF with transparent runtime status.</p>
621
  <div class="phase-badge-row">
622
  <span class="phase-badge"><strong>Model</strong> build-small-hackathon/phase-3-gguf</span>
623
+ <span class="phase-badge"><strong>Runtime</strong> llama.cpp server</span>
624
  <span class="phase-badge"><strong>Mode</strong> real GGUF inference</span>
625
  </div>
626
  </div>