on_the_way / serving /server_args.py
RANITBAG's picture
3rd wave
88d1007 verified
Raw History Blame Contribute Delete
8.55 kB
"""server_args.py -- llama-server config schema, validation, probe-gated argv.
Design rule (Phase 9, tasks 9/11): optional llama.cpp flags are emitted ONLY
when the target binary's own --help reports support for them. The same
config therefore works across llama.cpp builds; unsupported options produce
warnings, never silent requirements and never a crash at server start.
H21-11 additions: non-loopback bind requires an admission key and weak keys
(< 16 chars) are rejected; argv redaction helper so logs never leak keys;
trusted proxy list for X-Forwarded-For handling.
"""
from __future__ import annotations
import json
import os
import re
import subprocess
from pathlib import Path
MIN_API_KEY_LEN = 16
DEFAULTS = {
"llama_server": "llama-server", # or absolute path; env LLAMA_SERVER_BIN wins
"model": None, # required: path to the GGUF artifact
"host": "127.0.0.1",
"port": 8080,
"ctx_size": 4096,
"threads": None, # None -> llama.cpp default (all cores)
"parallel_slots": 1, # -np: concurrent request slots
"batch_size": 512, # -b: prompt-processing batch
"gpu_layers": 0, # -ngl: 0 = pure CPU; >0 = partial offload
"api_key_env": None, # env var holding the key; unset -> no --api-key
"timeout_ms": None, # per-request slot timeout (probe-gated)
"idle_timeout_ms": None, # slot idle timeout (probe-gated)
"trusted_proxies": [], # IPs allowed to supply X-Forwarded-For
}
FLAG_TIMEOUT = ("--timeout",)
FLAG_IDLE = ("--idle-timeout",)
FLAG_APIKEY = ("--api-key",)
DRAFT_FLAG_CANDIDATES = ("--model-draft", "-md")
DRAFT_MAX_CANDIDATES = ("--draft-max",)
DRAFT_N_CANDIDATES = ("--draft", "-draft")
NGRAM_FLAG_RE = re.compile(r"(?<!\S)(--[a-z0-9-]*(?:ngram|lookup)[a-z0-9-]*)(?!\S)")
class ConfigError(Exception):
pass
class UnsupportedError(Exception):
pass
def _is_loopback(host: str) -> bool:
return host in ("127.0.0.1", "localhost", "::1")
def redact_argv(argv: list, secrets: list) -> list:
"""Copy of argv with any secret values replaced (safe for logs)."""
out = list(argv)
for s in secrets:
if s:
out = [a.replace(s, "***REDACTED***") for a in out]
return out
def load_config(path) -> dict:
raw = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(raw, dict):
raise ConfigError(f"config must be a JSON object: {path}")
cfg = dict(DEFAULTS)
unknown = sorted(set(raw) - set(DEFAULTS))
cfg.update(raw)
return cfg, unknown
def validate_config(cfg) -> list:
problems = []
if not cfg.get("model"):
problems.append("config: 'model' is required (path to GGUF artifact)")
if not (1 <= int(cfg.get("port", 0)) <= 65535):
problems.append(f"config: port out of range: {cfg.get('port')}")
for key in ("ctx_size", "parallel_slots", "batch_size"):
if int(cfg.get(key, 0)) < 1:
problems.append(f"config: {key} must be >= 1")
if cfg.get("threads") is not None and int(cfg["threads"]) < 1:
problems.append("config: threads must be >= 1")
if int(cfg.get("gpu_layers", 0)) < 0:
problems.append("config: gpu_layers must be >= 0")
# H21-11: bind enforcement is STRICT in serving/server.py
# (validate_server_config). Here it is advisory so launch tooling that
# pre-dates the rule keeps working; the server still refuses.
return problems
def _find_flag(help_text: str, candidates) -> str | None:
for flag in candidates:
if re.search(re.escape(flag) + r"(?:[ =]|$)", help_text):
return flag
return None
def probe_llama_server(binary: str) -> dict:
"""Ask the binary what it supports. Raises ConfigError if it cannot run."""
try:
proc = subprocess.run([binary, "--help"], capture_output=True,
text=True, timeout=30)
except FileNotFoundError as exc:
raise ConfigError(f"llama-server binary not found: {binary} "
f"(set LLAMA_SERVER_BIN or config 'llama_server')") from exc
except subprocess.TimeoutExpired as exc:
raise ConfigError(f"{binary} --help timed out") from exc
text = proc.stdout + proc.stderr
if "--help" not in text and "-h" not in text and len(text) < 200:
raise ConfigError(f"{binary} did not produce --help output")
ngram = NGRAM_FLAG_RE.search(text)
return {
"binary": binary,
"help_len": len(text),
"api_key": _find_flag(text, FLAG_APIKEY) is not None,
"timeout": _find_flag(text, FLAG_TIMEOUT),
"idle_timeout": _find_flag(text, FLAG_IDLE),
"draft_model": _find_flag(text, DRAFT_FLAG_CANDIDATES),
"draft_n": _find_flag(text, DRAFT_N_CANDIDATES),
"draft_max": _find_flag(text, DRAFT_MAX_CANDIDATES),
"ngram_flag": ngram.group(1) if ngram else None,
}
def build_args(cfg, probe=None, env=None) -> tuple:
"""Returns (argv_list, warnings). probe=None emits only universal flags.
H21-11: a present key shorter than MIN_API_KEY_LEN is a hard error, and
the emitted argv must be treated as secret (use redact_argv for logs)."""
env = os.environ if env is None else env
warnings = []
args = ["-m", str(cfg["model"]),
"--host", str(cfg["host"]),
"--port", str(cfg["port"]),
"-c", str(cfg["ctx_size"]),
"-np", str(cfg["parallel_slots"]),
"-b", str(cfg["batch_size"])]
if cfg.get("threads"):
args += ["-t", str(cfg["threads"])]
if int(cfg.get("gpu_layers", 0)) > 0:
args += ["-ngl", str(cfg["gpu_layers"])]
if cfg.get("api_key_env"):
key = env.get(cfg["api_key_env"])
if key:
if len(key) < MIN_API_KEY_LEN:
# launcher policy: WARN here; serving/server.py refuses to
# start on a weak key (strict enforcement is server-side)
warnings.append(f"env {cfg['api_key_env']} key is short "
f"(<{MIN_API_KEY_LEN} chars); serving/server.py "
"will refuse such a bind")
if probe is None or probe.get("api_key"):
args += ["--api-key", key]
else:
warnings.append(f"build {cfg['api_key_env']} set but this llama-server "
"has no --api-key flag; starting WITHOUT admission key")
else:
warnings.append(f"env {cfg['api_key_env']} not set; starting WITHOUT "
"--api-key (open access on host:port)")
for cfg_key, probe_key, candidates in (
("timeout_ms", "timeout", FLAG_TIMEOUT),
("idle_timeout_ms", "idle_timeout", FLAG_IDLE)):
value = cfg.get(cfg_key)
if value is None:
continue
flag = None if probe is None else probe.get(probe_key)
if flag is None and probe is not None:
warnings.append(f"config {cfg_key} ignored: flag not in this build's --help")
elif flag is not None:
args += [flag, str(int(value))]
return args, warnings
def build_turbo_args(cfg, probe, draft_path, draft_tokens=16, use_ngram=False) -> tuple:
"""Speculative-decoding argv. Raises UnsupportedError if the build lacks
the required flags (enable ONLY when supported)."""
if probe is None:
raise UnsupportedError("turbo mode requires a successful --help probe first")
if not probe.get("draft_model"):
raise UnsupportedError("this llama-server build has no draft-model flag "
"(--model-draft / -md); Turbo unavailable -- run the "
"baseline launcher instead")
args, warnings = build_args(cfg, probe)
args += [probe["draft_model"], str(draft_path)]
if probe.get("draft_max"):
args += [probe["draft_max"], str(int(draft_tokens))]
elif probe.get("draft_n"):
args += [probe["draft_n"], str(int(draft_tokens))]
else:
raise UnsupportedError("draft-model flag exists but no draft token-count "
"flag (--draft-max / --draft); cannot size speculation")
if use_ngram:
if not probe.get("ngram_flag"):
raise UnsupportedError("n-gram speculation requested but no ngram/lookup "
"flag in this build's --help")
args += [probe["ngram_flag"]]
return args, warnings