bpHigh's picture
Upload via Autoresearch export_data
287ca83 verified
Raw History Blame Contribute Delete
1.51 kB
"""``python -m glance_runtime serve`` (HTTP) or ``python -m glance_runtime decide`` (one request from a file)."""
from __future__ import annotations
import argparse
import json
import sys
from glance_runtime.infer import DEFAULT_MODEL, Glance, make_server
def main(argv: list[str] | None = None) -> None:
ap = argparse.ArgumentParser(prog="glance-runtime")
sub = ap.add_subparsers(dest="cmd", required=True)
for name in ("serve", "decide"):
p = sub.add_parser(name)
p.add_argument("--model", default=DEFAULT_MODEL, help="Hugging Face repo id or local folder")
p.add_argument("--device", default="cpu", help="cpu, cuda or cuda:N")
p.add_argument("--dtype", default=None, help="bfloat16 (default), float16 or float32")
if name == "serve":
p.add_argument("--host", default="127.0.0.1")
p.add_argument("--port", type=int, default=8741)
else:
p.add_argument("request", help="a JSON request file, or - for stdin")
a = ap.parse_args(argv)
glance = Glance.load(a.model, device=a.device, dtype=a.dtype)
if a.cmd == "serve":
server = make_server(glance, a.host, a.port)
print(f"serving {glance.name} on http://{a.host}:{a.port}/v1/systemone", file=sys.stderr, flush=True)
server.serve_forever()
else:
request = json.load(sys.stdin if a.request == "-" else open(a.request))
print(json.dumps(glance.decide(request), indent=1))
if __name__ == "__main__":
main()