neojev-mlx / server.py
dowands's picture
Add files using upload-large-folder tool
74630d5 verified
Raw History Blame Contribute Delete
7.93 kB
"""TypeSafe-compatible HTTP server for NeoJev: POST /v1/systemone, GET /health (stdlib only).
python server.py --backend mlx --model dowands/neojev-mlx --port 8000
python server.py --backend cuda --model dowands/neojev --port 8000
Request: {"state": str | object, "questions": {name: {"type": "choice"|"score"|"noul", "instructions": str,
"criteria": ...}}, "image": base64 PNG/JPEG (optional), "image_max_side": int (optional)}
choice criteria: {label: description | null} (or a list of labels)
score criteria: [level description, ...] (labels are "0", "1", ...)
noul criteria: {"true": ..., "false": ...} (optional)
Response: {"model": ..., "answers": {name: {"type": ..., "choice"|"score"|"noul": ..., "probabilities": {...},
"confidence": ...}}, "usage": {"input_tokens": n}}
With "image" the screenshot becomes part of the state ("state" may then be omitted) and the questions-first
layout is always used for that request (the layout the head was fitted and timed in); it is scaled so its long
side is at most image_max_side (default --image-max-side, 32 to 2048) and image tokens count in input_tokens.
--layout state_first (default) puts the state before the questions and caches nothing; it is the layout used for
the published JevBench numbers. --layout schema_first caches the question prefix per schema (LRU), which is faster
when the same questions are asked about many states.
"""
import argparse
import base64
import binascii
import json
from http.server import BaseHTTPRequestHandler, HTTPServer
MIN_SIDE, MAX_SIDE_LIMIT = 32, 2048 # same limits as vision_common.py
def translate(questions, native_noul=False):
"""TypeSafe question dict -> (NeoJev question dict, per-question (type, labels, reversed)).
native_noul (used for image requests): a noul without true/false criteria becomes NeoJev's own noul, whose
probabilities come back as [Yes, No], hence reversed=True. Text requests keep the published mapping.
"""
out, labels = {}, {}
for name, q in questions.items():
t, crit, ins = q["type"], q.get("criteria"), q.get("instructions", "")
if t == "choice":
if isinstance(crit, dict):
labs = list(crit)
opts = [f"{l}: {crit[l]}" if crit[l] else l for l in labs]
else:
labs = [str(x) for x in (crit or [])]
opts = labs
out[name] = {"type": "choice", "options": opts, "prompt": ins}
elif t == "noul" and native_noul and not crit:
labs = ["no", "yes"]
out[name] = {"type": "noul", "statement": ins}
labels[name] = (t, labs, True)
continue
elif t == "noul":
c = crit or {}
labs = ["no", "yes"]
out[name] = {"type": "choice", "prompt": ins,
"options": [f"no: {c['false']}" if c.get("false") else "no",
f"yes: {c['true']}" if c.get("true") else "yes"]}
elif t == "score":
levels = [str(x) for x in (crit or [])]
labs = [str(i) for i in range(len(levels))]
out[name] = {"type": "score", "levels": levels, "prompt": ins}
else:
raise ValueError(f"unknown question type {t!r}")
labels[name] = (t, labs, False)
return out, labels
def parse_request(req, default_max_side):
"""JSON body -> (state text, NeoJev questions, labels, image bytes or None, image_max_side). Raises on bad input."""
image = req.get("image")
if image is None:
st = req["state"]
else:
st = req.get("state") or ""
s = "".join((image.split(",", 1)[1] if image.startswith("data:") else image).split()) # allow wrapped lines
try:
image = base64.b64decode(s, validate=True)
except (binascii.Error, ValueError) as e:
raise ValueError(f"image is not valid base64: {e}") from None
state = st if isinstance(st, str) else json.dumps(st, ensure_ascii=False, indent=1)
max_side = req.get("image_max_side", default_max_side)
if isinstance(max_side, bool) or not isinstance(max_side, int) or not MIN_SIDE <= max_side <= MAX_SIDE_LIMIT:
raise ValueError(f"image_max_side must be an integer between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
qs, labels = translate(req["questions"], native_noul=image is not None)
return state, qs, labels, image, max_side
def make_handler(d, layout, model_name, default_max_side):
class H(BaseHTTPRequestHandler):
def _send(self, code, obj):
b = json.dumps(obj).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(b)))
self.end_headers()
self.wfile.write(b)
def log_message(self, *args):
pass
def do_GET(self):
self._send(200, {"ok": True, "model": model_name}) if self.path == "/health" else self._send(404, {"error": "not found"})
def do_POST(self):
if self.path != "/v1/systemone":
return self._send(404, {"error": "not found"})
try:
req = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
state, qs, labels, image, max_side = parse_request(req, default_max_side)
kw = {} if image is None else {"image": image, "image_max_side": max_side}
# images always use the cached-questions layout: the head and the 1 s target were validated there
res = d.ask(state, qs, cache_schema=image is not None or layout == "schema_first", **kw)
except Exception as e: # malformed request
return self._send(400, {"error": f"{type(e).__name__}: {e}"})
answers = {}
for name, (t, labs, rev) in labels.items():
p = res[name]["probs_list"][::-1] if rev else res[name]["probs_list"]
probs = dict(zip(labs, p))
j = max(range(len(p)), key=lambda k: p[k])
ans = {"type": t, "probabilities": probs, "confidence": p[j]}
if t == "choice":
ans["choice"] = labs[j]
elif t == "noul":
ans["noul"] = probs["yes"]
else:
ans["score"] = sum(k * v for k, v in enumerate(p))
answers[name] = ans
self._send(200, {"model": model_name, "answers": answers, "usage": {"input_tokens": res["_tokens"]}})
return H
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--backend", choices=["mlx", "cuda"], required=True)
ap.add_argument("--model", required=True)
ap.add_argument("--port", type=int, default=8000)
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--layout", choices=["state_first", "schema_first"], default="state_first")
ap.add_argument("--token-budget", type=int, default=10 ** 6, help="cap on tokens per call (default: no truncation)")
ap.add_argument("--image-max-side", type=int, default=576, help="default long side images are scaled to (576 keeps an MLX call under 1 s on an M5 Pro)")
a = ap.parse_args()
if a.backend == "mlx":
from decider import Decider
d = Decider(a.model, token_budget=a.token_budget)
else:
from decider_cuda import Decider
d = Decider(a.model, token_budget=a.token_budget)
model_name = "neojev-qwen3.8-27b-l56"
print(f"NeoJev ({a.backend}, {a.layout}) on http://{a.host}:{a.port}/v1/systemone", flush=True)
# single-threaded on purpose: one GPU serves one request at a time, and MLX streams are bound to the main thread
HTTPServer((a.host, a.port), make_handler(d, a.layout, model_name, a.image_max_side)).serve_forever()
if __name__ == "__main__":
main()