Download serving/launch_server.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 2.8 kB
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/launch_server.py
- Command line
-
hf download hf://RANITBAG/on_the_way/serving/launch_server.py
-
curl -L -o launch_server.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/serving/launch_server.py
2.8 kB
| #!/usr/bin/env python3 | |
| """launch_server.py -- start llama-server from a Phase-9 serving config. | |
| Python only builds argv and spawns llama-server; all concurrency and decode | |
| work stay inside llama-server (task 6: no Python worker on the hot path). | |
| Usage: | |
| python serving/launch_server.py --config serving/config/q4_km_cpu.json | |
| python serving/launch_server.py --config ... --dry-run # print argv only | |
| python serving/launch_server.py --config ... --turbo --draft draft.gguf \ | |
| --draft-tokens 16 [--ngram] | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| import shlex | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from serving.server_args import ( # noqa: E402 | |
| ConfigError, UnsupportedError, build_args, build_turbo_args, | |
| load_config, probe_llama_server, validate_config, | |
| ) | |
| def main(argv=None) -> int: | |
| p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) | |
| p.add_argument("--config", required=True, type=Path) | |
| p.add_argument("--dry-run", action="store_true", help="print argv, do not start") | |
| p.add_argument("--turbo", action="store_true", | |
| help="enable speculative decoding (ONLY if the build supports it)") | |
| p.add_argument("--draft", type=Path, default=None, | |
| help="draft model GGUF (required with --turbo)") | |
| p.add_argument("--draft-tokens", type=int, default=16) | |
| p.add_argument("--ngram", action="store_true", | |
| help="n-gram speculation fallback (only if the build supports it)") | |
| args = p.parse_args(argv) | |
| try: | |
| cfg, unknown = load_config(args.config) | |
| problems = validate_config(cfg) | |
| if problems: | |
| raise ConfigError("; ".join(problems)) | |
| binary = os.environ.get("LLAMA_SERVER_BIN", cfg["llama_server"]) | |
| probe = probe_llama_server(binary) | |
| if args.turbo: | |
| if not args.draft: | |
| raise ConfigError("--turbo requires --draft <model.gguf>") | |
| argv_out, warnings = build_turbo_args( | |
| cfg, probe, args.draft, args.draft_tokens, args.ngram) | |
| else: | |
| argv_out, warnings = build_args(cfg, probe) | |
| except (ConfigError, UnsupportedError) as exc: | |
| print(f"ERROR: {exc}", file=sys.stderr) | |
| return 2 | |
| for w in unknown and [f"ignoring unknown config keys: {unknown}"] or []: | |
| print(f"WARNING: {w}", file=sys.stderr) | |
| for w in warnings: | |
| print(f"WARNING: {w}", file=sys.stderr) | |
| cmd = [binary, *argv_out] | |
| print("launching:", shlex.join(cmd)) | |
| if args.dry_run: | |
| return 0 | |
| try: | |
| return subprocess.run(cmd).returncode | |
| except KeyboardInterrupt: | |
| return 130 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |