Serve GGUF with persistent llama.cpp server
Browse files- README.md +5 -5
- __pycache__/app.cpython-314.pyc +0 -0
- app.py +139 -67
README.md
CHANGED
|
@@ -28,7 +28,7 @@ license: mit
|
|
| 28 |
|
| 29 |
First-Principle AI is a compact Gradio console for running and probing the
|
| 30 |
`build-small-hackathon/phase-3-gguf` Q8 GGUF model through
|
| 31 |
-
the official `llama.cpp` Ubuntu
|
| 32 |
|
| 33 |
The UI includes benchmark-style examples inspired by common LLM evaluation
|
| 34 |
areas: math reasoning, commonsense, science QA, truthfulness, instruction
|
|
@@ -39,12 +39,12 @@ questions are original prompts, not copied benchmark items.
|
|
| 39 |
|
| 40 |
- Model repo: `build-small-hackathon/phase-3-gguf`
|
| 41 |
- Model file: `model-Q8_0.gguf`
|
| 42 |
-
- Runtime: official `llama.cpp` `llama-
|
| 43 |
- Hardware target: ZeroGPU
|
| 44 |
- Fallback behavior: visible runtime diagnostics instead of silent mock output
|
| 45 |
-
- Model loading: runtime download/load through `llama-
|
| 46 |
-
- Default llama.cpp settings: `n_ctx=
|
| 47 |
-
memory-mapped weights, and CPU fallback if CUDA offload is unavailable
|
| 48 |
|
| 49 |
ZeroGPU is a Gradio dynamic GPU runtime primarily documented around PyTorch
|
| 50 |
workloads. This app targets ZeroGPU as requested, but it runs the GGUF through
|
|
|
|
| 28 |
|
| 29 |
First-Principle AI is a compact Gradio console for running and probing the
|
| 30 |
`build-small-hackathon/phase-3-gguf` Q8 GGUF model through
|
| 31 |
+
the official `llama.cpp` Ubuntu `llama-server` release.
|
| 32 |
|
| 33 |
The UI includes benchmark-style examples inspired by common LLM evaluation
|
| 34 |
areas: math reasoning, commonsense, science QA, truthfulness, instruction
|
|
|
|
| 39 |
|
| 40 |
- Model repo: `build-small-hackathon/phase-3-gguf`
|
| 41 |
- Model file: `model-Q8_0.gguf`
|
| 42 |
+
- Runtime: official `llama.cpp` `llama-server`
|
| 43 |
- Hardware target: ZeroGPU
|
| 44 |
- Fallback behavior: visible runtime diagnostics instead of silent mock output
|
| 45 |
+
- Model loading: runtime download/load through a persistent `llama-server`
|
| 46 |
+
- Default llama.cpp settings: `n_ctx=2048`, `n_batch=256`, `n_ubatch=64`,
|
| 47 |
+
memory-mapped weights, no warmup, and CPU fallback if CUDA offload is unavailable
|
| 48 |
|
| 49 |
ZeroGPU is a Gradio dynamic GPU runtime primarily documented around PyTorch
|
| 50 |
workloads. This app targets ZeroGPU as requested, but it runs the GGUF through
|
__pycache__/app.cpython-314.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-314.pyc and b/__pycache__/app.cpython-314.pyc differ
|
|
|
app.py
CHANGED
|
@@ -8,6 +8,7 @@ import time
|
|
| 8 |
import subprocess
|
| 9 |
import tarfile
|
| 10 |
import urllib.request
|
|
|
|
| 11 |
from pathlib import Path
|
| 12 |
from typing import Any
|
| 13 |
|
|
@@ -28,12 +29,12 @@ LLAMA_URL = os.getenv(
|
|
| 28 |
"PHASE3_LLAMA_URL",
|
| 29 |
f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_RELEASE}/llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz",
|
| 30 |
)
|
| 31 |
-
MAX_CONTEXT = int(os.getenv("PHASE3_MAX_CONTEXT", "
|
| 32 |
MIN_RAM_GB = float(os.getenv("PHASE3_MIN_RAM_GB", "38"))
|
| 33 |
DISABLE_MODEL = os.getenv("PHASE3_DISABLE_MODEL", "").lower() in {"1", "true", "yes"}
|
| 34 |
USE_ZEROGPU_DECORATOR = os.getenv("PHASE3_USE_ZEROGPU", "").lower() in {"1", "true", "yes"}
|
| 35 |
-
N_BATCH = int(os.getenv("PHASE3_N_BATCH", "
|
| 36 |
-
N_UBATCH = int(os.getenv("PHASE3_N_UBATCH", "
|
| 37 |
N_THREADS = int(os.getenv("PHASE3_THREADS", str(max(1, min(16, os.cpu_count() or 2)))))
|
| 38 |
N_THREADS_BATCH = int(os.getenv("PHASE3_THREADS_BATCH", str(N_THREADS)))
|
| 39 |
USE_MMAP = os.getenv("PHASE3_USE_MMAP", "1").lower() not in {"0", "false", "no"}
|
|
@@ -41,10 +42,15 @@ USE_MLOCK = os.getenv("PHASE3_USE_MLOCK", "").lower() in {"1", "true", "yes"}
|
|
| 41 |
FLASH_ATTN = os.getenv("PHASE3_FLASH_ATTN", "").lower() in {"1", "true", "yes"}
|
| 42 |
OFFLOAD_KQV = os.getenv("PHASE3_OFFLOAD_KQV", "1").lower() not in {"0", "false", "no"}
|
| 43 |
INFER_TIMEOUT = int(os.getenv("PHASE3_INFER_TIMEOUT", "900"))
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
MODEL_LOCK = threading.Lock()
|
| 46 |
MODEL_PATH: Path | None = None
|
| 47 |
LLAMA_CLI_PATH: Path | None = None
|
|
|
|
|
|
|
| 48 |
MODEL_ERROR: str | None = None
|
| 49 |
MODEL_SETTINGS: dict[str, Any] = {}
|
| 50 |
|
|
@@ -151,18 +157,24 @@ def _gpu_layers() -> int:
|
|
| 151 |
return 0
|
| 152 |
|
| 153 |
|
| 154 |
-
def
|
| 155 |
-
global LLAMA_CLI_PATH
|
| 156 |
|
| 157 |
-
if LLAMA_CLI_PATH is not None and LLAMA_CLI_PATH.exists():
|
| 158 |
return LLAMA_CLI_PATH
|
|
|
|
|
|
|
| 159 |
|
| 160 |
root = Path(os.getenv("PHASE3_LLAMA_DIR", "/tmp/phase3-llama.cpp"))
|
| 161 |
release_dir = root / f"llama-{LLAMA_RELEASE}"
|
| 162 |
-
|
| 163 |
-
if
|
| 164 |
-
|
| 165 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
|
| 167 |
root.mkdir(parents=True, exist_ok=True)
|
| 168 |
archive = root / f"llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz"
|
|
@@ -170,11 +182,14 @@ def _ensure_llama_cli() -> Path:
|
|
| 170 |
urllib.request.urlretrieve(LLAMA_URL, archive)
|
| 171 |
with tarfile.open(archive, "r:gz") as tar:
|
| 172 |
tar.extractall(root)
|
| 173 |
-
if not
|
| 174 |
-
raise RuntimeError(f"
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
|
|
|
|
|
|
|
|
|
| 178 |
|
| 179 |
|
| 180 |
def _prepare_runtime() -> tuple[Path, Path]:
|
|
@@ -196,12 +211,12 @@ def _prepare_runtime() -> tuple[Path, Path]:
|
|
| 196 |
raise RuntimeError(MODEL_ERROR)
|
| 197 |
|
| 198 |
path = _find_model_path()
|
| 199 |
-
|
| 200 |
MODEL_PATH = path
|
| 201 |
n_gpu_layers = _gpu_layers()
|
| 202 |
MODEL_SETTINGS = {
|
| 203 |
"path": str(path),
|
| 204 |
-
"
|
| 205 |
"n_ctx": MAX_CONTEXT,
|
| 206 |
"n_batch": N_BATCH,
|
| 207 |
"n_ubatch": N_UBATCH,
|
|
@@ -212,41 +227,50 @@ def _prepare_runtime() -> tuple[Path, Path]:
|
|
| 212 |
"use_mlock": USE_MLOCK,
|
| 213 |
"flash_attn": FLASH_ATTN,
|
| 214 |
"offload_kqv": OFFLOAD_KQV,
|
|
|
|
| 215 |
}
|
| 216 |
-
return path,
|
| 217 |
|
| 218 |
|
| 219 |
-
def
|
| 220 |
-
|
| 221 |
-
turns = [f"<|im_start|>system\n{system}<|im_end|>"]
|
| 222 |
-
for item in history[-10:]:
|
| 223 |
-
role = item.get("role", "user")
|
| 224 |
-
content = item.get("content", "")
|
| 225 |
-
if role in {"user", "assistant"} and content:
|
| 226 |
-
turns.append(f"<|im_start|>{role}\n{content}<|im_end|>")
|
| 227 |
-
turns.append(f"<|im_start|>user\n{message}<|im_end|>")
|
| 228 |
-
turns.append("<|im_start|>assistant\n")
|
| 229 |
-
return "\n".join(turns)
|
| 230 |
|
| 231 |
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
cmd = [
|
| 243 |
-
str(
|
| 244 |
"-m",
|
| 245 |
str(model_path),
|
| 246 |
-
"-
|
| 247 |
-
|
| 248 |
-
"-
|
| 249 |
-
str(
|
| 250 |
"-c",
|
| 251 |
str(MAX_CONTEXT),
|
| 252 |
"-t",
|
|
@@ -255,13 +279,6 @@ def _complete(
|
|
| 255 |
str(N_BATCH),
|
| 256 |
"-ub",
|
| 257 |
str(N_UBATCH),
|
| 258 |
-
"--temp",
|
| 259 |
-
str(float(temperature)),
|
| 260 |
-
"--top-p",
|
| 261 |
-
str(float(top_p)),
|
| 262 |
-
"--repeat-penalty",
|
| 263 |
-
str(float(repeat_penalty)),
|
| 264 |
-
"--no-display-prompt",
|
| 265 |
]
|
| 266 |
if _gpu_layers() != 0:
|
| 267 |
cmd.extend(["-ngl", str(_gpu_layers())])
|
|
@@ -271,25 +288,79 @@ def _complete(
|
|
| 271 |
cmd.append("--no-mmap")
|
| 272 |
if FLASH_ATTN:
|
| 273 |
cmd.append("-fa")
|
|
|
|
|
|
|
| 274 |
|
| 275 |
env = os.environ.copy()
|
| 276 |
-
binary_dir = str(
|
| 277 |
env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
cmd,
|
| 280 |
cwd=binary_dir,
|
| 281 |
env=env,
|
|
|
|
|
|
|
| 282 |
text=True,
|
| 283 |
-
capture_output=True,
|
| 284 |
-
timeout=INFER_TIMEOUT,
|
| 285 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
elapsed = max(time.time() - started, 0.001)
|
| 287 |
-
|
| 288 |
-
stderr = proc.stderr.strip()
|
| 289 |
-
stdout = proc.stdout.strip()
|
| 290 |
-
detail = stderr or stdout or f"llama-cli exited with code {proc.returncode}"
|
| 291 |
-
raise RuntimeError(detail[-4000:])
|
| 292 |
-
text = proc.stdout.strip()
|
| 293 |
text = text.split("<|im_end|>", 1)[0].strip()
|
| 294 |
completion_tokens = max(1, len(text.split()))
|
| 295 |
return text, {
|
|
@@ -308,7 +379,8 @@ def _status_markdown() -> str:
|
|
| 308 |
model_state = "Ready" if MODEL_PATH is not None else ("Error" if MODEL_ERROR else "Ready to load on first prompt")
|
| 309 |
available_text = f"{available_gb:.1f} GB" if available_gb is not None else "unknown"
|
| 310 |
path_text = f"`{MODEL_PATH}`" if MODEL_PATH else "not resolved yet"
|
| 311 |
-
|
|
|
|
| 312 |
settings = MODEL_SETTINGS or {
|
| 313 |
"n_ctx": MAX_CONTEXT,
|
| 314 |
"n_batch": N_BATCH,
|
|
@@ -335,11 +407,11 @@ def _status_markdown() -> str:
|
|
| 335 |
| Available RAM | {available_text} |
|
| 336 |
| CUDA devices | `{cuda_text}` |
|
| 337 |
| Model path | {path_text} |
|
| 338 |
-
| llama-
|
| 339 |
| llama.cpp settings | `ctx={settings.get('n_ctx')}`, `batch={settings.get('n_batch')}`, `ubatch={settings.get('n_ubatch')}`, `threads={settings.get('n_threads')}`, `gpu_layers={settings.get('n_gpu_layers')}` |
|
| 340 |
-
| Memory/options | `mmap={settings.get('use_mmap')}`, `mlock={settings.get('use_mlock')}`, `flash_attn={settings.get('flash_attn')}`, `
|
| 341 |
|
| 342 |
-
The first prompt
|
| 343 |
"""
|
| 344 |
|
| 345 |
|
|
@@ -396,7 +468,7 @@ def respond(
|
|
| 396 |
"Model load or inference failed.\n\n"
|
| 397 |
f"{exc}\n\n"
|
| 398 |
"The UI is live and the model artifact is published, but the runtime could not complete "
|
| 399 |
-
"a llama.cpp
|
| 400 |
)
|
| 401 |
meta = {"elapsed": 0.0, "completion_tokens": len(text.split()), "tokens_per_second": 0.0}
|
| 402 |
|
|
@@ -548,7 +620,7 @@ with gr.Blocks(title="First-Principle AI", fill_width=True) as demo:
|
|
| 548 |
<p>A clean model-console interface for probing the Phase-3 Q8 GGUF with transparent runtime status.</p>
|
| 549 |
<div class="phase-badge-row">
|
| 550 |
<span class="phase-badge"><strong>Model</strong> build-small-hackathon/phase-3-gguf</span>
|
| 551 |
-
<span class="phase-badge"><strong>Runtime</strong> llama.cpp
|
| 552 |
<span class="phase-badge"><strong>Mode</strong> real GGUF inference</span>
|
| 553 |
</div>
|
| 554 |
</div>
|
|
|
|
| 8 |
import subprocess
|
| 9 |
import tarfile
|
| 10 |
import urllib.request
|
| 11 |
+
import json
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Any
|
| 14 |
|
|
|
|
| 29 |
"PHASE3_LLAMA_URL",
|
| 30 |
f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_RELEASE}/llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz",
|
| 31 |
)
|
| 32 |
+
MAX_CONTEXT = int(os.getenv("PHASE3_MAX_CONTEXT", "2048"))
|
| 33 |
MIN_RAM_GB = float(os.getenv("PHASE3_MIN_RAM_GB", "38"))
|
| 34 |
DISABLE_MODEL = os.getenv("PHASE3_DISABLE_MODEL", "").lower() in {"1", "true", "yes"}
|
| 35 |
USE_ZEROGPU_DECORATOR = os.getenv("PHASE3_USE_ZEROGPU", "").lower() in {"1", "true", "yes"}
|
| 36 |
+
N_BATCH = int(os.getenv("PHASE3_N_BATCH", "256"))
|
| 37 |
+
N_UBATCH = int(os.getenv("PHASE3_N_UBATCH", "64"))
|
| 38 |
N_THREADS = int(os.getenv("PHASE3_THREADS", str(max(1, min(16, os.cpu_count() or 2)))))
|
| 39 |
N_THREADS_BATCH = int(os.getenv("PHASE3_THREADS_BATCH", str(N_THREADS)))
|
| 40 |
USE_MMAP = os.getenv("PHASE3_USE_MMAP", "1").lower() not in {"0", "false", "no"}
|
|
|
|
| 42 |
FLASH_ATTN = os.getenv("PHASE3_FLASH_ATTN", "").lower() in {"1", "true", "yes"}
|
| 43 |
OFFLOAD_KQV = os.getenv("PHASE3_OFFLOAD_KQV", "1").lower() not in {"0", "false", "no"}
|
| 44 |
INFER_TIMEOUT = int(os.getenv("PHASE3_INFER_TIMEOUT", "900"))
|
| 45 |
+
SERVER_HOST = "127.0.0.1"
|
| 46 |
+
SERVER_PORT = int(os.getenv("PHASE3_SERVER_PORT", "8088"))
|
| 47 |
+
NO_WARMUP = os.getenv("PHASE3_NO_WARMUP", "1").lower() not in {"0", "false", "no"}
|
| 48 |
|
| 49 |
MODEL_LOCK = threading.Lock()
|
| 50 |
MODEL_PATH: Path | None = None
|
| 51 |
LLAMA_CLI_PATH: Path | None = None
|
| 52 |
+
LLAMA_SERVER_PATH: Path | None = None
|
| 53 |
+
LLAMA_SERVER_PROCESS: subprocess.Popen[str] | None = None
|
| 54 |
MODEL_ERROR: str | None = None
|
| 55 |
MODEL_SETTINGS: dict[str, Any] = {}
|
| 56 |
|
|
|
|
| 157 |
return 0
|
| 158 |
|
| 159 |
|
| 160 |
+
def _ensure_llama_binary(name: str) -> Path:
|
| 161 |
+
global LLAMA_CLI_PATH, LLAMA_SERVER_PATH
|
| 162 |
|
| 163 |
+
if name == "llama-cli" and LLAMA_CLI_PATH is not None and LLAMA_CLI_PATH.exists():
|
| 164 |
return LLAMA_CLI_PATH
|
| 165 |
+
if name == "llama-server" and LLAMA_SERVER_PATH is not None and LLAMA_SERVER_PATH.exists():
|
| 166 |
+
return LLAMA_SERVER_PATH
|
| 167 |
|
| 168 |
root = Path(os.getenv("PHASE3_LLAMA_DIR", "/tmp/phase3-llama.cpp"))
|
| 169 |
release_dir = root / f"llama-{LLAMA_RELEASE}"
|
| 170 |
+
binary = release_dir / name
|
| 171 |
+
if binary.exists():
|
| 172 |
+
binary.chmod(0o755)
|
| 173 |
+
if name == "llama-cli":
|
| 174 |
+
LLAMA_CLI_PATH = binary
|
| 175 |
+
if name == "llama-server":
|
| 176 |
+
LLAMA_SERVER_PATH = binary
|
| 177 |
+
return binary
|
| 178 |
|
| 179 |
root.mkdir(parents=True, exist_ok=True)
|
| 180 |
archive = root / f"llama-{LLAMA_RELEASE}-bin-ubuntu-x64.tar.gz"
|
|
|
|
| 182 |
urllib.request.urlretrieve(LLAMA_URL, archive)
|
| 183 |
with tarfile.open(archive, "r:gz") as tar:
|
| 184 |
tar.extractall(root)
|
| 185 |
+
if not binary.exists():
|
| 186 |
+
raise RuntimeError(f"{name} was not found after extracting {LLAMA_URL}")
|
| 187 |
+
binary.chmod(0o755)
|
| 188 |
+
if name == "llama-cli":
|
| 189 |
+
LLAMA_CLI_PATH = binary
|
| 190 |
+
if name == "llama-server":
|
| 191 |
+
LLAMA_SERVER_PATH = binary
|
| 192 |
+
return binary
|
| 193 |
|
| 194 |
|
| 195 |
def _prepare_runtime() -> tuple[Path, Path]:
|
|
|
|
| 211 |
raise RuntimeError(MODEL_ERROR)
|
| 212 |
|
| 213 |
path = _find_model_path()
|
| 214 |
+
server = _ensure_llama_binary("llama-server")
|
| 215 |
MODEL_PATH = path
|
| 216 |
n_gpu_layers = _gpu_layers()
|
| 217 |
MODEL_SETTINGS = {
|
| 218 |
"path": str(path),
|
| 219 |
+
"llama_server": str(server),
|
| 220 |
"n_ctx": MAX_CONTEXT,
|
| 221 |
"n_batch": N_BATCH,
|
| 222 |
"n_ubatch": N_UBATCH,
|
|
|
|
| 227 |
"use_mlock": USE_MLOCK,
|
| 228 |
"flash_attn": FLASH_ATTN,
|
| 229 |
"offload_kqv": OFFLOAD_KQV,
|
| 230 |
+
"no_warmup": NO_WARMUP,
|
| 231 |
}
|
| 232 |
+
return path, server
|
| 233 |
|
| 234 |
|
| 235 |
+
def _server_log_path() -> Path:
|
| 236 |
+
return Path(os.getenv("PHASE3_SERVER_LOG", "/tmp/phase3-llama-server.log"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
|
| 238 |
|
| 239 |
+
def _tail_server_log(limit: int = 4000) -> str:
|
| 240 |
+
path = _server_log_path()
|
| 241 |
+
if not path.exists():
|
| 242 |
+
return ""
|
| 243 |
+
data = path.read_text(encoding="utf-8", errors="ignore")
|
| 244 |
+
return data[-limit:]
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _server_url(path: str) -> str:
|
| 248 |
+
return f"http://{SERVER_HOST}:{SERVER_PORT}{path}"
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def _server_is_ready() -> bool:
|
| 252 |
+
try:
|
| 253 |
+
with urllib.request.urlopen(_server_url("/health"), timeout=5) as resp:
|
| 254 |
+
return 200 <= resp.status < 500
|
| 255 |
+
except Exception:
|
| 256 |
+
return False
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _start_server() -> None:
|
| 260 |
+
global LLAMA_SERVER_PROCESS
|
| 261 |
+
|
| 262 |
+
model_path, server = _prepare_runtime()
|
| 263 |
+
if LLAMA_SERVER_PROCESS is not None and LLAMA_SERVER_PROCESS.poll() is None and _server_is_ready():
|
| 264 |
+
return
|
| 265 |
+
|
| 266 |
cmd = [
|
| 267 |
+
str(server),
|
| 268 |
"-m",
|
| 269 |
str(model_path),
|
| 270 |
+
"--host",
|
| 271 |
+
SERVER_HOST,
|
| 272 |
+
"--port",
|
| 273 |
+
str(SERVER_PORT),
|
| 274 |
"-c",
|
| 275 |
str(MAX_CONTEXT),
|
| 276 |
"-t",
|
|
|
|
| 279 |
str(N_BATCH),
|
| 280 |
"-ub",
|
| 281 |
str(N_UBATCH),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
]
|
| 283 |
if _gpu_layers() != 0:
|
| 284 |
cmd.extend(["-ngl", str(_gpu_layers())])
|
|
|
|
| 288 |
cmd.append("--no-mmap")
|
| 289 |
if FLASH_ATTN:
|
| 290 |
cmd.append("-fa")
|
| 291 |
+
if NO_WARMUP:
|
| 292 |
+
cmd.append("--no-warmup")
|
| 293 |
|
| 294 |
env = os.environ.copy()
|
| 295 |
+
binary_dir = str(server.parent)
|
| 296 |
env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
|
| 297 |
+
log_path = _server_log_path()
|
| 298 |
+
log_file = log_path.open("a", encoding="utf-8")
|
| 299 |
+
log_file.write(f"\n--- starting llama-server: {' '.join(cmd)} ---\n")
|
| 300 |
+
log_file.flush()
|
| 301 |
+
LLAMA_SERVER_PROCESS = subprocess.Popen(
|
| 302 |
cmd,
|
| 303 |
cwd=binary_dir,
|
| 304 |
env=env,
|
| 305 |
+
stdout=log_file,
|
| 306 |
+
stderr=subprocess.STDOUT,
|
| 307 |
text=True,
|
|
|
|
|
|
|
| 308 |
)
|
| 309 |
+
|
| 310 |
+
deadline = time.time() + INFER_TIMEOUT
|
| 311 |
+
while time.time() < deadline:
|
| 312 |
+
if LLAMA_SERVER_PROCESS.poll() is not None:
|
| 313 |
+
raise RuntimeError(f"llama-server exited early.\n{_tail_server_log()}")
|
| 314 |
+
if _server_is_ready():
|
| 315 |
+
return
|
| 316 |
+
time.sleep(2)
|
| 317 |
+
raise RuntimeError(f"llama-server did not become ready within {INFER_TIMEOUT}s.\n{_tail_server_log()}")
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def _format_prompt(system_prompt: str, history: list[dict[str, str]], message: str) -> str:
|
| 321 |
+
system = system_prompt.strip() or "You are a precise, direct model in a technical lab console."
|
| 322 |
+
turns = [f"<|im_start|>system\n{system}<|im_end|>"]
|
| 323 |
+
for item in history[-10:]:
|
| 324 |
+
role = item.get("role", "user")
|
| 325 |
+
content = item.get("content", "")
|
| 326 |
+
if role in {"user", "assistant"} and content:
|
| 327 |
+
turns.append(f"<|im_start|>{role}\n{content}<|im_end|>")
|
| 328 |
+
turns.append(f"<|im_start|>user\n{message}<|im_end|>")
|
| 329 |
+
turns.append("<|im_start|>assistant\n")
|
| 330 |
+
return "\n".join(turns)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
@_gpu_decorator
|
| 334 |
+
def _complete(
|
| 335 |
+
prompt: str,
|
| 336 |
+
max_tokens: int,
|
| 337 |
+
temperature: float,
|
| 338 |
+
top_p: float,
|
| 339 |
+
repeat_penalty: float,
|
| 340 |
+
) -> tuple[str, dict[str, Any]]:
|
| 341 |
+
started = time.time()
|
| 342 |
+
_start_server()
|
| 343 |
+
payload = {
|
| 344 |
+
"prompt": prompt,
|
| 345 |
+
"n_predict": int(max_tokens),
|
| 346 |
+
"temperature": float(temperature),
|
| 347 |
+
"top_p": float(top_p),
|
| 348 |
+
"repeat_penalty": float(repeat_penalty),
|
| 349 |
+
"stop": ["<|im_end|>", "<|endoftext|>"],
|
| 350 |
+
}
|
| 351 |
+
req = urllib.request.Request(
|
| 352 |
+
_server_url("/completion"),
|
| 353 |
+
data=json.dumps(payload).encode("utf-8"),
|
| 354 |
+
headers={"Content-Type": "application/json"},
|
| 355 |
+
method="POST",
|
| 356 |
+
)
|
| 357 |
+
try:
|
| 358 |
+
with urllib.request.urlopen(req, timeout=INFER_TIMEOUT) as resp:
|
| 359 |
+
output = json.loads(resp.read().decode("utf-8"))
|
| 360 |
+
except Exception as exc:
|
| 361 |
+
raise RuntimeError(f"llama-server completion failed: {exc}\n{_tail_server_log()}") from exc
|
| 362 |
elapsed = max(time.time() - started, 0.001)
|
| 363 |
+
text = (output.get("content") or "").strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 364 |
text = text.split("<|im_end|>", 1)[0].strip()
|
| 365 |
completion_tokens = max(1, len(text.split()))
|
| 366 |
return text, {
|
|
|
|
| 379 |
model_state = "Ready" if MODEL_PATH is not None else ("Error" if MODEL_ERROR else "Ready to load on first prompt")
|
| 380 |
available_text = f"{available_gb:.1f} GB" if available_gb is not None else "unknown"
|
| 381 |
path_text = f"`{MODEL_PATH}`" if MODEL_PATH else "not resolved yet"
|
| 382 |
+
server_text = f"`{LLAMA_SERVER_PATH}`" if LLAMA_SERVER_PATH else f"`{LLAMA_RELEASE}` not extracted yet"
|
| 383 |
+
server_state = "running" if LLAMA_SERVER_PROCESS is not None and LLAMA_SERVER_PROCESS.poll() is None else "not started"
|
| 384 |
settings = MODEL_SETTINGS or {
|
| 385 |
"n_ctx": MAX_CONTEXT,
|
| 386 |
"n_batch": N_BATCH,
|
|
|
|
| 407 |
| Available RAM | {available_text} |
|
| 408 |
| CUDA devices | `{cuda_text}` |
|
| 409 |
| Model path | {path_text} |
|
| 410 |
+
| llama-server | {server_text} ({server_state}) |
|
| 411 |
| llama.cpp settings | `ctx={settings.get('n_ctx')}`, `batch={settings.get('n_batch')}`, `ubatch={settings.get('n_ubatch')}`, `threads={settings.get('n_threads')}`, `gpu_layers={settings.get('n_gpu_layers')}` |
|
| 412 |
+
| Memory/options | `mmap={settings.get('use_mmap')}`, `mlock={settings.get('use_mlock')}`, `flash_attn={settings.get('flash_attn')}`, `no_warmup={settings.get('no_warmup')}` |
|
| 413 |
|
| 414 |
+
The first prompt starts `llama-server` and loads the 31 GB Q8 GGUF if it is not already cached. Later prompts reuse the same llama.cpp server process.
|
| 415 |
"""
|
| 416 |
|
| 417 |
|
|
|
|
| 468 |
"Model load or inference failed.\n\n"
|
| 469 |
f"{exc}\n\n"
|
| 470 |
"The UI is live and the model artifact is published, but the runtime could not complete "
|
| 471 |
+
"a llama.cpp server generation pass. Check the runtime status and Space logs before retrying."
|
| 472 |
)
|
| 473 |
meta = {"elapsed": 0.0, "completion_tokens": len(text.split()), "tokens_per_second": 0.0}
|
| 474 |
|
|
|
|
| 620 |
<p>A clean model-console interface for probing the Phase-3 Q8 GGUF with transparent runtime status.</p>
|
| 621 |
<div class="phase-badge-row">
|
| 622 |
<span class="phase-badge"><strong>Model</strong> build-small-hackathon/phase-3-gguf</span>
|
| 623 |
+
<span class="phase-badge"><strong>Runtime</strong> llama.cpp server</span>
|
| 624 |
<span class="phase-badge"><strong>Mode</strong> real GGUF inference</span>
|
| 625 |
</div>
|
| 626 |
</div>
|