File size: 14,915 Bytes
515b383
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
"""Micro-batching HTTP server.
  POST /decide        {"context": str, "schema": {...}}                     -> typed JSON decisions (all questions packed in one row)
  POST /v1/systemone  {"state": str|object|array, "questions": {id: {...}}} -> the TypeSafe/Jev wire format (decider.systemone):
                      Choice (up to 255 described options), Score, Noul; every question is scored in its own row, so answers
                      are independent of each other ("independent": false packs them behind one copy of the state instead).
Requests arriving within `max_wait_ms` are scored in one forward pass (grouped by length bucket).
   uvicorn decider.serve:app --host 0.0.0.0 --port 8000     (env: DECIDER_MODEL, DECIDER_MAX_BATCH, DECIDER_MAX_WAIT_MS)
"""
import asyncio, os, random, time, threading
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from decider.engine import Engine, T_BUCKETS, _bucket
from decider.prompt import build, MAX_OPTIONS
from decider.infer import Decider, Example, Q, neutralize_options
from decider import systemone as S1

MODEL = os.environ.get("DECIDER_MODEL", "runs/r3_v2/model")
MAX_BATCH = int(os.environ.get("DECIDER_MAX_BATCH", "32"))
MAX_WAIT_MS = float(os.environ.get("DECIDER_MAX_WAIT_MS", "8"))
BATCH_WAIT_MS = float(os.environ.get("DECIDER_BATCH_WAIT_MS", "0"))
MAX_STATE_TOKENS = int(os.environ.get("DECIDER_MAX_STATE_TOKENS", "32768"))
MAX_FWD_TOKENS = int(os.environ.get("DECIDER_MAX_FWD_TOKENS", "65536"))     # padded tokens per forward pass
COMPILE = os.environ.get("DECIDER_COMPILE", "1") == "1"
FP8 = os.environ.get("DECIDER_FP8", "1") == "1"
app = FastAPI(title="decider")
MODEL_NAME = "decider"; TEMP = 1.0; TEMP_SCHEMA = 1.0; RELEASE_DATE = "2026-09-17"
gpu_lock = threading.Lock()            # one GPU job at a time: batched graph replays and shared-prefix requests must not interleave
SHARED_MIN_TOKENS = int(os.environ.get("DECIDER_SHARED_MIN_TOKENS", "768"))   # independent rows over a state this long share one prefix pass
eng = None; queue = None; stats = dict(requests=0, batches=0, decisions=0, batch_hist={})


class Req(BaseModel):
    context: str
    schema_: dict = None
    model_config = {"populate_by_name": True}
    def __init__(self, **kw):
        if "schema" in kw: kw["schema_"] = kw.pop("schema")
        super().__init__(**kw)


class _NoShuffle:
    def shuffle(self, x): pass
    def sample(self, xs, k): return xs[:k]


def _prepare(context, schema):
    qs = Decider._schema_to_questions(schema)
    for q in qs:
        if getattr(eng, "neutralize_none", True):
            q["options"], q["_back"] = neutralize_options(q["options"])
    ex = Example(context, [Q(q["question"], list(q["options"]), 0) for q in qs])
    it = build(ex, eng.tok, _NoShuffle(), max_options=MAX_OPTIONS, max_ctx_tokens=eng.max_ctx)
    return qs, it


def _format(schema, qs, probs):
    o = {}
    for (qtext, spec), q, p in zip(schema.items(), qs, probs):
        p = p[:len(q["options"])].tolist(); t = spec.get("type", "choice"); j = max(range(len(p)), key=p.__getitem__)
        back = q.get("_back", {}); names = [back.get(x, x) for x in q["options"]]
        if t == "bool":
            o[qtext] = {"noul": round(p[1], 4), "type": "noul"}
        elif t == "choice":
            o[qtext] = {"choice": names[j], "confidence": round(p[j], 4), "type": "choice",
                        "probabilities": {k: round(v, 4) for k, v in zip(names, p)}}
        else:
            keys = q["_keys"]; score = sum(float(k) * pi for k, pi in zip(keys, p))
            o[qtext] = {"score": round(score, 2), "confidence": round(p[j], 4), "type": "scale", "legend": q["_legend"],
                        "probabilities": {str(keys[i]): round(pi, 4) for i, pi in enumerate(p)}}
    return o


async def _collect(q):
    """Continuous batching: take what is already queued and go.  While a forward pass runs, new requests pile up and form the
    next batch, so there is no fixed wait at low load (it cost 1.5 ms per request) and full batches at high load.
    DECIDER_BATCH_WAIT_MS > 0 restores a short collection window."""
    batch = [await q.get()]; deadline = time.monotonic() + BATCH_WAIT_MS / 1000
    while len(batch) < MAX_BATCH:
        try:
            batch.append(q.get_nowait())
        except asyncio.QueueEmpty:
            timeout = deadline - time.monotonic()
            if timeout <= 0: break
            try: batch.append(await asyncio.wait_for(q.get(), timeout))
            except asyncio.TimeoutError: break
    return batch


async def batcher():
    loop = asyncio.get_running_loop()
    while True:
        batch = await _collect(queue)
        # sort by length; split into at most two groups when the spread is large (keeps padding small)
        batch.sort(key=lambda x: len(x[2]["ids"]))
        groups = [batch]
        if len(batch) >= 4:
            lo, hi = len(batch[0][2]["ids"]), len(batch[-1][2]["ids"])
            if _bucket(hi, T_BUCKETS) != _bucket(lo, T_BUCKETS) and hi > 1.5 * lo:
                cut = len(batch) // 2; groups = [batch[:cut], batch[cut:]]
        capped = []                                       # long rows: keep every forward under MAX_FWD_TOKENS padded tokens
        for g in groups:
            cur = []
            for x in g:
                if cur and (len(cur) + 1) * len(x[2]["ids"]) > MAX_FWD_TOKENS:
                    capped.append(cur); cur = []
                cur.append(x)
            capped.append(cur)
        for g in capped:
            items = [it for _, _, it in g]
            try:
                probs = await loop.run_in_executor(None, _locked, eng.score_items, items)
                for (fut, qs, it), p in zip(g, probs):
                    if not fut.done(): fut.set_result(p)
            except Exception as e:
                for fut, _, _ in g:
                    if not fut.done(): fut.set_exception(e)
            stats["batches"] += 1; stats["batch_hist"][len(g)] = stats["batch_hist"].get(len(g), 0) + 1


@app.on_event("startup")
async def _start():
    global eng, queue
    eng = Engine(MODEL, compile=COMPILE, fp8=FP8, conv_patch=COMPILE); print("[serve] engine", eng.cfg, flush=True)
    import json
    global MODEL_NAME, TEMP
    try: cfg = json.load(open(os.path.join(MODEL, "decider_config.json")))
    except Exception: cfg = {}
    eng.neutralize_none = bool(cfg.get("neutralize_none", True)); MODEL_NAME = "decider-" + str(cfg.get("version", "dev"))
    TEMP = float(os.environ.get("DECIDER_TEMPERATURE", cfg.get("temperature", 1.0)))
    global RELEASE_DATE; RELEASE_DATE = str(cfg.get("release_date", RELEASE_DATE))
    global SCHEMA_FIRST, se, squeue, ISOLATED
    ISOLATED = bool(cfg.get("isolated_levels", False))
    # the schema cache needs the questions-first layout, which costs accuracy (about 1.5 points on fixed label sets, more on large
    # label sets and long states): on when the model's config makes it the default, or with DECIDER_SCHEMA_CACHE=1
    trained = bool(cfg.get("schema_first", False) or cfg.get("schema_first_trained", False))
    SCHEMA_FIRST = trained and (bool(cfg.get("schema_first", False)) or os.environ.get("DECIDER_SCHEMA_CACHE", "0") == "1")
    global TEMP_SCHEMA; TEMP_SCHEMA = float(cfg.get("temperature_schema_first", TEMP))
    if SCHEMA_FIRST:
        from decider.schema_engine import SchemaEngine
        se = SchemaEngine(eng); squeue = asyncio.Queue(); asyncio.create_task(schema_batcher()); print("[serve] schema cache on", flush=True)
        pre = os.environ.get("DECIDER_SCHEMAS")             # JSON file: [{"questions": {...}, "independent": true, "batch_sizes": [1, 8, 32], "state_tokens": [64, 256]}]
        for spec in (json.load(open(pre)) if pre else []):  # known schemas: prefix computed, graphs compiled and captured before traffic
            _, h, _ = _schema_handle(spec["questions"], spec.get("independent", True), compile=COMPILE)
            t = se.warmup(h, spec.get("batch_sizes", (1, 8, 32)), spec.get("state_tokens", (64, 128, 256)))
            print(f"[serve] preloaded schema with {h.nq} rows, prefix {sum(h.tps)} tokens, graphs ready in {t:.0f}s", flush=True)
    shapes = [(B, T) for B in (1, 2, 4, 8, 16, 32) for T in T_BUCKETS if T <= eng.max_ctx + 256]
    if MAX_BATCH > 32: shapes += [(64, T) for T in T_BUCKETS if T <= 512]
    t = eng.warmup(shapes); print(f"[serve] captured {len(shapes)} graphs in {t:.0f}s", flush=True)
    queue = asyncio.Queue()
    asyncio.create_task(batcher())


@app.post("/decide")
async def decide(r: Req):
    qs, it = await asyncio.get_running_loop().run_in_executor(None, _prepare, r.context, r.schema_)
    fut = asyncio.get_running_loop().create_future()
    await queue.put((fut, qs, it))
    probs = await fut
    stats["requests"] += 1; stats["decisions"] += len(qs)
    return _format(r.schema_, qs, probs)


SCHEMA_FIRST = False; se = None; squeue = None; schemas = {}            # schema cache (models trained on the questions-first layout, v7+)


ISOLATED = False


def _schema_handle(questions, independent, compile=False):
    """Compile (or look up) the question schema: its prefix is run once, requests then only run the state."""
    import json
    key = (json.dumps(questions, sort_keys=True, ensure_ascii=False), independent)
    if key not in schemas:
        rqs = {k: S1.render_question(v) for k, v in questions.items()}; rows, index = S1.plan_rows(rqs, ISOLATED and independent)
        with gpu_lock:
            if len(schemas) >= 128:
                old = next(iter(schemas)); hid = schemas.pop(old)[1].id
                for k in [k for k in se.graphs if k[0] == hid]: del se.graphs[k]
            h = se.prepare(rows, independent=independent, compile=compile)
        schemas[key] = (rqs, h, index)
    return schemas[key]


seen = {}


def _worth_caching(questions, independent):
    """A schema gets a cached prefix and CUDA graphs from its second request on: one-off schemas go through the generic
    state-first engine, whose graphs do not depend on the questions, so ad-hoc traffic cannot thrash graph captures."""
    import json
    key = (json.dumps(questions, sort_keys=True, ensure_ascii=False), independent)
    if key in schemas: return True
    if len(seen) > 50000: seen.clear()
    seen[key] = seen.get(key, 0) + 1
    return seen[key] >= int(os.environ.get("DECIDER_SCHEMA_MIN_SEEN", "2"))


def _score_schema(h, rows):
    with gpu_lock:
        return se.score_rows(h, rows, temperature=TEMP_SCHEMA)


async def schema_batcher():
    """Requests that share a schema and arrive within the window are scored in one forward pass over their states."""
    loop = asyncio.get_running_loop()
    while True:
        batch = await _collect(squeue)
        groups = {}                                        # one forward per (schema, length bucket): short states are not padded to long ones
        for fut, h, row in batch: groups.setdefault((h.id, se.bucket(len(row[0]))), (h, []))[1].append((fut, row))
        for h, items in groups.values():
            step = max(1, MAX_BATCH // h.P)
            for i in range(0, len(items), step):
                chunk = items[i:i + step]
                try:
                    probs = await loop.run_in_executor(None, _score_schema, h, [c for _, c in chunk])
                    for (fut, _), p in zip(chunk, probs):
                        if not fut.done(): fut.set_result(p)
                except Exception as e:
                    for fut, _ in chunk:
                        if not fut.done(): fut.set_exception(e)
            stats["schema_batches"] = stats.get("schema_batches", 0) + 1


def _locked(fn, items):
    with gpu_lock:
        return fn(items, temperature=TEMP)          # fitted temperature from decider_config.json


class S1Req(BaseModel):
    state: object
    questions: dict
    model: str | None = None
    independent: bool = True
    layout: str | None = None            # "state_first" forces the uncached layout on a schema-first model


def _prepare_s1(state, questions, independent):
    ctx = S1.render_state(state); rqs = {k: S1.render_question(v) for k, v in questions.items()}
    flat, index = S1.plan_rows(rqs, ISOLATED and independent)
    rows = [[r] for r in flat] if independent else [flat]
    items = [build(Example(ctx, [Q(r["question"], list(r["options"]), 0) for r in row]), eng.tok, _NoShuffle(), max_options=MAX_OPTIONS,
                   max_ctx_tokens=MAX_STATE_TOKENS) for row in rows]
    return (rqs, index), items


@app.post("/v1/systemone")
async def systemone(r: S1Req):
    loop = asyncio.get_running_loop()
    if SCHEMA_FIRST and r.layout != "state_first" and _worth_caching(r.questions, r.independent):
        try:
            rqs, h, index = await loop.run_in_executor(None, _schema_handle, r.questions, r.independent)
        except ValueError as e:
            raise HTTPException(422, str(e))
        row = await loop.run_in_executor(None, lambda: se.tokenize(h, S1.render_state(r.state), MAX_STATE_TOKENS))      # CPU work stays off the GPU lock
        fut = loop.create_future(); await squeue.put((fut, h, row)); p = await fut
        stats["requests"] += 1; stats["decisions"] += len(rqs); stats["schema_requests"] = stats.get("schema_requests", 0) + 1
        return {"model": MODEL_NAME, "answers": S1.assemble(rqs, index, [pk.tolist() for pk in p]),
                "usage": {"input_tokens": len(row[0]) * h.P, "cached_tokens": sum(h.tps), "output_tokens": 0}}
    try:
        (rqs, index), items = await loop.run_in_executor(None, _prepare_s1, r.state, r.questions, r.independent)
    except ValueError as e:
        raise HTTPException(422, str(e))
    if len(items) > 1 and min(len(it["ids"]) for it in items) >= SHARED_MIN_TOKENS:
        res = await loop.run_in_executor(None, _locked, eng.score_shared, items)      # long state: run it once, fork the cache per question
        stats["shared_prefix_requests"] = stats.get("shared_prefix_requests", 0) + 1
    else:
        futs = []
        for it in items:
            f = loop.create_future(); futs.append(f); await queue.put((f, None, it))
        res = await asyncio.gather(*futs)
    probs = [p for ps in res for p in ps]                                   # one prob row per question, request order
    stats["requests"] += 1; stats["decisions"] += len(rqs)
    return {"model": MODEL_NAME, "answers": S1.assemble(rqs, index, [p.tolist() for p in probs]),
            "usage": {"input_tokens": S1.unique_tokens(items), "output_tokens": 0}}


@app.get("/v1/models")
async def models():
    return {"models": [{"name": MODEL_NAME, "description": "decider: one-pass typed decisions with calibrated probabilities", "release_date": RELEASE_DATE}]}


@app.get("/health")
async def health():
    return {"ok": eng is not None, "model": MODEL}


@app.get("/stats")
async def get_stats():
    return dict(stats, engine=eng.stats if eng else None, graphs=len(eng.graphs) if eng else 0)