Spaces:
Running
Running
File size: 4,903 Bytes
0a2e8cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """`openjev serve | calibrate | bench | version`."""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import sys
from pathlib import Path
from . import __version__
from .backends import make_backend
from .config import PROFILES_DIR, Settings, load_profile
from .readout import Readout
def _serve(a):
s = Settings(**{k: v for k, v in vars(a).items() if v is not None and k in Settings.model_fields})
logging.basicConfig(level=s.log_level, format="%(message)s")
profile = load_profile(
s.profile, {"perms": a.perms, "temp": a.temp, "noul_t": a.noul_t, "assistant_prefix": a.assistant_prefix, "letter_prefix": a.letter_prefix}
)
try:
backend, extra = make_backend(s, profile)
except ValueError as e:
sys.exit(str(e))
readout = Readout(backend, profile)
from .backends.probe import probe
found = asyncio.run(backend.start())
report = asyncio.run(
probe(readout, backend, letter_prefix_used=found.get("letter_prefix", ""), exact=found.get("exact_readout"), vision=found.get("vision"))
)
found.update(report)
logging.getLogger("openjev").info(json.dumps({"model_probe": found}))
if "problem" in report and not a.force:
sys.exit(f"refusing to serve: {report['problem']} (start with --force to serve anyway)")
import uvicorn
from .server import build_app
app = build_app(readout, token=s.token, model_dir=s.model, backend_name=s.backend, extra_version={**extra, "model_probe": found})
logging.getLogger("openjev").info(
json.dumps({"serving": f"http://{s.host}:{s.port}/v1/systemone", "backend": s.backend, "model": s.model, "profile": profile.to_dict()})
)
uvicorn.run(app, host=s.host, port=s.port, log_level="warning", access_log=False)
def _calibrate(a):
from .calibrate import collect, fit, write_profile
with open(a.dev) as f:
rows = [json.loads(line) for line in f if line.strip()]
collected = asyncio.run(collect(rows, a.endpoint, a.token, a.concurrency))
result = fit(collected)
print(json.dumps(result, indent=1))
base = Path(a.base) if Path(a.base).exists() else PROFILES_DIR / f"{a.base}.json"
out = Path(a.out)
write_profile(base, result, out.stem, out)
print(f"profile written: {out} (serve it with --profile {out})")
def main(argv=None):
ap = argparse.ArgumentParser(prog="openjev", description=f"openjev-server {__version__}: a decision API over any open model")
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("serve", help="serve POST /v1/systemone")
s.add_argument("--backend", choices=["vllm", "mlx"])
s.add_argument("--model", help="tokenizer / model dir (vllm) or the MLX model dir (mlx)")
s.add_argument("--vllm-url", dest="vllm_url")
s.add_argument("--served-model-name", dest="served_model_name")
s.add_argument("--host")
s.add_argument("--port", type=int)
s.add_argument("--token")
s.add_argument("--profile", help="profile name under profiles/ or a JSON path")
s.add_argument("--perms", type=int, help="average over N option orders (accuracy mode; costs latency)")
s.add_argument("--temp", type=float)
s.add_argument("--noul-t", dest="noul_t", type=float)
s.add_argument("--no-prefix-cache", dest="prefix_cache", action="store_false", default=None)
s.add_argument(
"--assistant-prefix",
dest="assistant_prefix",
help="text at the start of the assistant turn before the readout (e.g. an empty think block for models that always reason first)",
)
s.add_argument("--letter-prefix", dest="letter_prefix", choices=["auto", "", " "], help="label token form: bare 'A', space-prefixed ' A', or auto")
s.add_argument("--force", action="store_true", help="serve even when the startup probe reports a problem")
s.set_defaults(fn=_serve)
c = sub.add_parser("calibrate", help="fit temperature and yes/no scale on development rows against a running server")
c.add_argument("--dev", required=True, help="JSONL: {state, questions: {id: question}, gold}")
c.add_argument("--endpoint", default="http://localhost:3000")
c.add_argument("--token", default="")
c.add_argument("--base", default="uncalibrated", help="profile the server is running")
c.add_argument("--out", required=True, help="where to write the fitted profile JSON")
c.add_argument("--concurrency", type=int, default=8)
c.set_defaults(fn=_calibrate)
b = sub.add_parser("bench", help="latency / throughput of a running server")
b.set_defaults(fn=lambda a: __import__("openjev_server.bench", fromlist=["main"]).main(a.rest))
b.add_argument("rest", nargs=argparse.REMAINDER)
v = sub.add_parser("version")
v.set_defaults(fn=lambda a: print(__version__))
a = ap.parse_args(argv)
a.fn(a)
if __name__ == "__main__":
main()
|