File size: 7,928 Bytes
c4eeba4
 
74630d5
 
c4eeba4
 
74630d5
c4eeba4
 
 
 
 
 
74630d5
 
 
 
c4eeba4
 
 
 
 
74630d5
 
c4eeba4
 
 
74630d5
 
 
 
 
c4eeba4
74630d5
 
 
c4eeba4
 
 
 
 
 
 
 
 
 
 
74630d5
 
 
 
 
c4eeba4
 
 
 
 
 
 
 
 
 
 
 
74630d5
c4eeba4
 
 
74630d5
 
 
 
 
c4eeba4
74630d5
 
 
 
 
 
 
 
 
 
 
 
c4eeba4
74630d5
 
c4eeba4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74630d5
 
 
 
c4eeba4
 
 
74630d5
 
c4eeba4
 
 
 
 
 
 
 
 
 
 
 
74630d5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
c4eeba4
 
74630d5
c4eeba4
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""TypeSafe-compatible HTTP server for NeoJev: POST /v1/systemone, GET /health (stdlib only).

    python server.py --backend mlx  --model dowands/neojev-mlx --port 8000
    python server.py --backend cuda --model dowands/neojev      --port 8000

Request:  {"state": str | object, "questions": {name: {"type": "choice"|"score"|"noul", "instructions": str,
           "criteria": ...}}, "image": base64 PNG/JPEG (optional), "image_max_side": int (optional)}
  choice criteria: {label: description | null}   (or a list of labels)
  score  criteria: [level description, ...]      (labels are "0", "1", ...)
  noul   criteria: {"true": ..., "false": ...}   (optional)
Response: {"model": ..., "answers": {name: {"type": ..., "choice"|"score"|"noul": ..., "probabilities": {...},
           "confidence": ...}}, "usage": {"input_tokens": n}}

With "image" the screenshot becomes part of the state ("state" may then be omitted) and the questions-first
layout is always used for that request (the layout the head was fitted and timed in); it is scaled so its long
side is at most image_max_side (default --image-max-side, 32 to 2048) and image tokens count in input_tokens.

--layout state_first (default) puts the state before the questions and caches nothing; it is the layout used for
the published JevBench numbers. --layout schema_first caches the question prefix per schema (LRU), which is faster
when the same questions are asked about many states.
"""
import argparse
import base64
import binascii
import json
from http.server import BaseHTTPRequestHandler, HTTPServer

MIN_SIDE, MAX_SIDE_LIMIT = 32, 2048   # same limits as vision_common.py


def translate(questions, native_noul=False):
    """TypeSafe question dict -> (NeoJev question dict, per-question (type, labels, reversed)).

    native_noul (used for image requests): a noul without true/false criteria becomes NeoJev's own noul, whose
    probabilities come back as [Yes, No], hence reversed=True. Text requests keep the published mapping.
    """
    out, labels = {}, {}
    for name, q in questions.items():
        t, crit, ins = q["type"], q.get("criteria"), q.get("instructions", "")
        if t == "choice":
            if isinstance(crit, dict):
                labs = list(crit)
                opts = [f"{l}: {crit[l]}" if crit[l] else l for l in labs]
            else:
                labs = [str(x) for x in (crit or [])]
                opts = labs
            out[name] = {"type": "choice", "options": opts, "prompt": ins}
        elif t == "noul" and native_noul and not crit:
            labs = ["no", "yes"]
            out[name] = {"type": "noul", "statement": ins}
            labels[name] = (t, labs, True)
            continue
        elif t == "noul":
            c = crit or {}
            labs = ["no", "yes"]
            out[name] = {"type": "choice", "prompt": ins,
                         "options": [f"no: {c['false']}" if c.get("false") else "no",
                                     f"yes: {c['true']}" if c.get("true") else "yes"]}
        elif t == "score":
            levels = [str(x) for x in (crit or [])]
            labs = [str(i) for i in range(len(levels))]
            out[name] = {"type": "score", "levels": levels, "prompt": ins}
        else:
            raise ValueError(f"unknown question type {t!r}")
        labels[name] = (t, labs, False)
    return out, labels


def parse_request(req, default_max_side):
    """JSON body -> (state text, NeoJev questions, labels, image bytes or None, image_max_side). Raises on bad input."""
    image = req.get("image")
    if image is None:
        st = req["state"]
    else:
        st = req.get("state") or ""
        s = "".join((image.split(",", 1)[1] if image.startswith("data:") else image).split())  # allow wrapped lines
        try:
            image = base64.b64decode(s, validate=True)
        except (binascii.Error, ValueError) as e:
            raise ValueError(f"image is not valid base64: {e}") from None
    state = st if isinstance(st, str) else json.dumps(st, ensure_ascii=False, indent=1)
    max_side = req.get("image_max_side", default_max_side)
    if isinstance(max_side, bool) or not isinstance(max_side, int) or not MIN_SIDE <= max_side <= MAX_SIDE_LIMIT:
        raise ValueError(f"image_max_side must be an integer between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
    qs, labels = translate(req["questions"], native_noul=image is not None)
    return state, qs, labels, image, max_side


def make_handler(d, layout, model_name, default_max_side):
    class H(BaseHTTPRequestHandler):
        def _send(self, code, obj):
            b = json.dumps(obj).encode()
            self.send_response(code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(b)))
            self.end_headers()
            self.wfile.write(b)

        def log_message(self, *args):
            pass

        def do_GET(self):
            self._send(200, {"ok": True, "model": model_name}) if self.path == "/health" else self._send(404, {"error": "not found"})

        def do_POST(self):
            if self.path != "/v1/systemone":
                return self._send(404, {"error": "not found"})
            try:
                req = json.loads(self.rfile.read(int(self.headers.get("Content-Length", 0))))
                state, qs, labels, image, max_side = parse_request(req, default_max_side)
                kw = {} if image is None else {"image": image, "image_max_side": max_side}
                # images always use the cached-questions layout: the head and the 1 s target were validated there
                res = d.ask(state, qs, cache_schema=image is not None or layout == "schema_first", **kw)
            except Exception as e:  # malformed request
                return self._send(400, {"error": f"{type(e).__name__}: {e}"})
            answers = {}
            for name, (t, labs, rev) in labels.items():
                p = res[name]["probs_list"][::-1] if rev else res[name]["probs_list"]
                probs = dict(zip(labs, p))
                j = max(range(len(p)), key=lambda k: p[k])
                ans = {"type": t, "probabilities": probs, "confidence": p[j]}
                if t == "choice":
                    ans["choice"] = labs[j]
                elif t == "noul":
                    ans["noul"] = probs["yes"]
                else:
                    ans["score"] = sum(k * v for k, v in enumerate(p))
                answers[name] = ans
            self._send(200, {"model": model_name, "answers": answers, "usage": {"input_tokens": res["_tokens"]}})

    return H


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--backend", choices=["mlx", "cuda"], required=True)
    ap.add_argument("--model", required=True)
    ap.add_argument("--port", type=int, default=8000)
    ap.add_argument("--host", default="127.0.0.1")
    ap.add_argument("--layout", choices=["state_first", "schema_first"], default="state_first")
    ap.add_argument("--token-budget", type=int, default=10 ** 6, help="cap on tokens per call (default: no truncation)")
    ap.add_argument("--image-max-side", type=int, default=576, help="default long side images are scaled to (576 keeps an MLX call under 1 s on an M5 Pro)")
    a = ap.parse_args()
    if a.backend == "mlx":
        from decider import Decider
        d = Decider(a.model, token_budget=a.token_budget)
    else:
        from decider_cuda import Decider
        d = Decider(a.model, token_budget=a.token_budget)
    model_name = "neojev-qwen3.8-27b-l56"
    print(f"NeoJev ({a.backend}, {a.layout}) on http://{a.host}:{a.port}/v1/systemone", flush=True)
    # single-threaded on purpose: one GPU serves one request at a time, and MLX streams are bound to the main thread
    HTTPServer((a.host, a.port), make_handler(d, a.layout, model_name, a.image_max_side)).serve_forever()


if __name__ == "__main__":
    main()