#!/usr/bin/env python3 """launch_server.py -- start llama-server from a Phase-9 serving config. Python only builds argv and spawns llama-server; all concurrency and decode work stay inside llama-server (task 6: no Python worker on the hot path). Usage: python serving/launch_server.py --config serving/config/q4_km_cpu.json python serving/launch_server.py --config ... --dry-run # print argv only python serving/launch_server.py --config ... --turbo --draft draft.gguf \ --draft-tokens 16 [--ngram] """ from __future__ import annotations import argparse import os import shlex import subprocess import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from serving.server_args import ( # noqa: E402 ConfigError, UnsupportedError, build_args, build_turbo_args, load_config, probe_llama_server, validate_config, ) def main(argv=None) -> int: p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) p.add_argument("--config", required=True, type=Path) p.add_argument("--dry-run", action="store_true", help="print argv, do not start") p.add_argument("--turbo", action="store_true", help="enable speculative decoding (ONLY if the build supports it)") p.add_argument("--draft", type=Path, default=None, help="draft model GGUF (required with --turbo)") p.add_argument("--draft-tokens", type=int, default=16) p.add_argument("--ngram", action="store_true", help="n-gram speculation fallback (only if the build supports it)") args = p.parse_args(argv) try: cfg, unknown = load_config(args.config) problems = validate_config(cfg) if problems: raise ConfigError("; ".join(problems)) binary = os.environ.get("LLAMA_SERVER_BIN", cfg["llama_server"]) probe = probe_llama_server(binary) if args.turbo: if not args.draft: raise ConfigError("--turbo requires --draft ") argv_out, warnings = build_turbo_args( cfg, probe, args.draft, args.draft_tokens, args.ngram) else: argv_out, warnings = build_args(cfg, probe) except (ConfigError, UnsupportedError) as exc: print(f"ERROR: {exc}", file=sys.stderr) return 2 for w in unknown and [f"ignoring unknown config keys: {unknown}"] or []: print(f"WARNING: {w}", file=sys.stderr) for w in warnings: print(f"WARNING: {w}", file=sys.stderr) cmd = [binary, *argv_out] print("launching:", shlex.join(cmd)) if args.dry_run: return 0 try: return subprocess.run(cmd).returncode except KeyboardInterrupt: return 130 if __name__ == "__main__": raise SystemExit(main())