on_the_way / serving /launch_server.py
RANITBAG's picture
2nd run
1a1fc1f verified
Raw History Blame Contribute Delete
2.8 kB
#!/usr/bin/env python3
"""launch_server.py -- start llama-server from a Phase-9 serving config.
Python only builds argv and spawns llama-server; all concurrency and decode
work stay inside llama-server (task 6: no Python worker on the hot path).
Usage:
python serving/launch_server.py --config serving/config/q4_km_cpu.json
python serving/launch_server.py --config ... --dry-run # print argv only
python serving/launch_server.py --config ... --turbo --draft draft.gguf \
--draft-tokens 16 [--ngram]
"""
from __future__ import annotations
import argparse
import os
import shlex
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from serving.server_args import ( # noqa: E402
ConfigError, UnsupportedError, build_args, build_turbo_args,
load_config, probe_llama_server, validate_config,
)
def main(argv=None) -> int:
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
p.add_argument("--config", required=True, type=Path)
p.add_argument("--dry-run", action="store_true", help="print argv, do not start")
p.add_argument("--turbo", action="store_true",
help="enable speculative decoding (ONLY if the build supports it)")
p.add_argument("--draft", type=Path, default=None,
help="draft model GGUF (required with --turbo)")
p.add_argument("--draft-tokens", type=int, default=16)
p.add_argument("--ngram", action="store_true",
help="n-gram speculation fallback (only if the build supports it)")
args = p.parse_args(argv)
try:
cfg, unknown = load_config(args.config)
problems = validate_config(cfg)
if problems:
raise ConfigError("; ".join(problems))
binary = os.environ.get("LLAMA_SERVER_BIN", cfg["llama_server"])
probe = probe_llama_server(binary)
if args.turbo:
if not args.draft:
raise ConfigError("--turbo requires --draft <model.gguf>")
argv_out, warnings = build_turbo_args(
cfg, probe, args.draft, args.draft_tokens, args.ngram)
else:
argv_out, warnings = build_args(cfg, probe)
except (ConfigError, UnsupportedError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2
for w in unknown and [f"ignoring unknown config keys: {unknown}"] or []:
print(f"WARNING: {w}", file=sys.stderr)
for w in warnings:
print(f"WARNING: {w}", file=sys.stderr)
cmd = [binary, *argv_out]
print("launching:", shlex.join(cmd))
if args.dry_run:
return 0
try:
return subprocess.run(cmd).returncode
except KeyboardInterrupt:
return 130
if __name__ == "__main__":
raise SystemExit(main())