Download serving/server_args.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 8.55 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/server_args.py
- Command line
-
hf download hf://RANITBAG/on_the_way/serving/server_args.py
-
curl -L -o server_args.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/server_args.py
8.55 kB
| """server_args.py -- llama-server config schema, validation, probe-gated argv. | |
| Design rule (Phase 9, tasks 9/11): optional llama.cpp flags are emitted ONLY | |
| when the target binary's own --help reports support for them. The same | |
| config therefore works across llama.cpp builds; unsupported options produce | |
| warnings, never silent requirements and never a crash at server start. | |
| H21-11 additions: non-loopback bind requires an admission key and weak keys | |
| (< 16 chars) are rejected; argv redaction helper so logs never leak keys; | |
| trusted proxy list for X-Forwarded-For handling. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import re | |
| import subprocess | |
| from pathlib import Path | |
| MIN_API_KEY_LEN = 16 | |
| DEFAULTS = { | |
| "llama_server": "llama-server", # or absolute path; env LLAMA_SERVER_BIN wins | |
| "model": None, # required: path to the GGUF artifact | |
| "host": "127.0.0.1", | |
| "port": 8080, | |
| "ctx_size": 4096, | |
| "threads": None, # None -> llama.cpp default (all cores) | |
| "parallel_slots": 1, # -np: concurrent request slots | |
| "batch_size": 512, # -b: prompt-processing batch | |
| "gpu_layers": 0, # -ngl: 0 = pure CPU; >0 = partial offload | |
| "api_key_env": None, # env var holding the key; unset -> no --api-key | |
| "timeout_ms": None, # per-request slot timeout (probe-gated) | |
| "idle_timeout_ms": None, # slot idle timeout (probe-gated) | |
| "trusted_proxies": [], # IPs allowed to supply X-Forwarded-For | |
| } | |
| FLAG_TIMEOUT = ("--timeout",) | |
| FLAG_IDLE = ("--idle-timeout",) | |
| FLAG_APIKEY = ("--api-key",) | |
| DRAFT_FLAG_CANDIDATES = ("--model-draft", "-md") | |
| DRAFT_MAX_CANDIDATES = ("--draft-max",) | |
| DRAFT_N_CANDIDATES = ("--draft", "-draft") | |
| NGRAM_FLAG_RE = re.compile(r"(?<!\S)(--[a-z0-9-]*(?:ngram|lookup)[a-z0-9-]*)(?!\S)") | |
| class ConfigError(Exception): | |
| pass | |
| class UnsupportedError(Exception): | |
| pass | |
| def _is_loopback(host: str) -> bool: | |
| return host in ("127.0.0.1", "localhost", "::1") | |
| def redact_argv(argv: list, secrets: list) -> list: | |
| """Copy of argv with any secret values replaced (safe for logs).""" | |
| out = list(argv) | |
| for s in secrets: | |
| if s: | |
| out = [a.replace(s, "***REDACTED***") for a in out] | |
| return out | |
| def load_config(path) -> dict: | |
| raw = json.loads(Path(path).read_text(encoding="utf-8")) | |
| if not isinstance(raw, dict): | |
| raise ConfigError(f"config must be a JSON object: {path}") | |
| cfg = dict(DEFAULTS) | |
| unknown = sorted(set(raw) - set(DEFAULTS)) | |
| cfg.update(raw) | |
| return cfg, unknown | |
| def validate_config(cfg) -> list: | |
| problems = [] | |
| if not cfg.get("model"): | |
| problems.append("config: 'model' is required (path to GGUF artifact)") | |
| if not (1 <= int(cfg.get("port", 0)) <= 65535): | |
| problems.append(f"config: port out of range: {cfg.get('port')}") | |
| for key in ("ctx_size", "parallel_slots", "batch_size"): | |
| if int(cfg.get(key, 0)) < 1: | |
| problems.append(f"config: {key} must be >= 1") | |
| if cfg.get("threads") is not None and int(cfg["threads"]) < 1: | |
| problems.append("config: threads must be >= 1") | |
| if int(cfg.get("gpu_layers", 0)) < 0: | |
| problems.append("config: gpu_layers must be >= 0") | |
| # H21-11: bind enforcement is STRICT in serving/server.py | |
| # (validate_server_config). Here it is advisory so launch tooling that | |
| # pre-dates the rule keeps working; the server still refuses. | |
| return problems | |
| def _find_flag(help_text: str, candidates) -> str | None: | |
| for flag in candidates: | |
| if re.search(re.escape(flag) + r"(?:[ =]|$)", help_text): | |
| return flag | |
| return None | |
| def probe_llama_server(binary: str) -> dict: | |
| """Ask the binary what it supports. Raises ConfigError if it cannot run.""" | |
| try: | |
| proc = subprocess.run([binary, "--help"], capture_output=True, | |
| text=True, timeout=30) | |
| except FileNotFoundError as exc: | |
| raise ConfigError(f"llama-server binary not found: {binary} " | |
| f"(set LLAMA_SERVER_BIN or config 'llama_server')") from exc | |
| except subprocess.TimeoutExpired as exc: | |
| raise ConfigError(f"{binary} --help timed out") from exc | |
| text = proc.stdout + proc.stderr | |
| if "--help" not in text and "-h" not in text and len(text) < 200: | |
| raise ConfigError(f"{binary} did not produce --help output") | |
| ngram = NGRAM_FLAG_RE.search(text) | |
| return { | |
| "binary": binary, | |
| "help_len": len(text), | |
| "api_key": _find_flag(text, FLAG_APIKEY) is not None, | |
| "timeout": _find_flag(text, FLAG_TIMEOUT), | |
| "idle_timeout": _find_flag(text, FLAG_IDLE), | |
| "draft_model": _find_flag(text, DRAFT_FLAG_CANDIDATES), | |
| "draft_n": _find_flag(text, DRAFT_N_CANDIDATES), | |
| "draft_max": _find_flag(text, DRAFT_MAX_CANDIDATES), | |
| "ngram_flag": ngram.group(1) if ngram else None, | |
| } | |
| def build_args(cfg, probe=None, env=None) -> tuple: | |
| """Returns (argv_list, warnings). probe=None emits only universal flags. | |
| H21-11: a present key shorter than MIN_API_KEY_LEN is a hard error, and | |
| the emitted argv must be treated as secret (use redact_argv for logs).""" | |
| env = os.environ if env is None else env | |
| warnings = [] | |
| args = ["-m", str(cfg["model"]), | |
| "--host", str(cfg["host"]), | |
| "--port", str(cfg["port"]), | |
| "-c", str(cfg["ctx_size"]), | |
| "-np", str(cfg["parallel_slots"]), | |
| "-b", str(cfg["batch_size"])] | |
| if cfg.get("threads"): | |
| args += ["-t", str(cfg["threads"])] | |
| if int(cfg.get("gpu_layers", 0)) > 0: | |
| args += ["-ngl", str(cfg["gpu_layers"])] | |
| if cfg.get("api_key_env"): | |
| key = env.get(cfg["api_key_env"]) | |
| if key: | |
| if len(key) < MIN_API_KEY_LEN: | |
| # launcher policy: WARN here; serving/server.py refuses to | |
| # start on a weak key (strict enforcement is server-side) | |
| warnings.append(f"env {cfg['api_key_env']} key is short " | |
| f"(<{MIN_API_KEY_LEN} chars); serving/server.py " | |
| "will refuse such a bind") | |
| if probe is None or probe.get("api_key"): | |
| args += ["--api-key", key] | |
| else: | |
| warnings.append(f"build {cfg['api_key_env']} set but this llama-server " | |
| "has no --api-key flag; starting WITHOUT admission key") | |
| else: | |
| warnings.append(f"env {cfg['api_key_env']} not set; starting WITHOUT " | |
| "--api-key (open access on host:port)") | |
| for cfg_key, probe_key, candidates in ( | |
| ("timeout_ms", "timeout", FLAG_TIMEOUT), | |
| ("idle_timeout_ms", "idle_timeout", FLAG_IDLE)): | |
| value = cfg.get(cfg_key) | |
| if value is None: | |
| continue | |
| flag = None if probe is None else probe.get(probe_key) | |
| if flag is None and probe is not None: | |
| warnings.append(f"config {cfg_key} ignored: flag not in this build's --help") | |
| elif flag is not None: | |
| args += [flag, str(int(value))] | |
| return args, warnings | |
| def build_turbo_args(cfg, probe, draft_path, draft_tokens=16, use_ngram=False) -> tuple: | |
| """Speculative-decoding argv. Raises UnsupportedError if the build lacks | |
| the required flags (enable ONLY when supported).""" | |
| if probe is None: | |
| raise UnsupportedError("turbo mode requires a successful --help probe first") | |
| if not probe.get("draft_model"): | |
| raise UnsupportedError("this llama-server build has no draft-model flag " | |
| "(--model-draft / -md); Turbo unavailable -- run the " | |
| "baseline launcher instead") | |
| args, warnings = build_args(cfg, probe) | |
| args += [probe["draft_model"], str(draft_path)] | |
| if probe.get("draft_max"): | |
| args += [probe["draft_max"], str(int(draft_tokens))] | |
| elif probe.get("draft_n"): | |
| args += [probe["draft_n"], str(int(draft_tokens))] | |
| else: | |
| raise UnsupportedError("draft-model flag exists but no draft token-count " | |
| "flag (--draft-max / --draft); cannot size speculation") | |
| if use_ngram: | |
| if not probe.get("ngram_flag"): | |
| raise UnsupportedError("n-gram speculation requested but no ngram/lookup " | |
| "flag in this build's --help") | |
| args += [probe["ngram_flag"]] | |
| return args, warnings | |