File size: 13,118 Bytes
24ef3cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74630d5
 
 
 
24ef3cd
 
74630d5
24ef3cd
 
 
 
 
74630d5
 
24ef3cd
 
 
c4eeba4
24ef3cd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74630d5
24ef3cd
 
 
 
 
 
74630d5
 
 
 
 
 
 
24ef3cd
 
 
 
74630d5
 
 
 
24ef3cd
 
 
 
 
 
 
 
 
 
 
 
 
c4eeba4
 
24ef3cd
74630d5
 
 
 
 
 
24ef3cd
 
 
74630d5
 
24ef3cd
 
 
 
74630d5
 
 
 
 
 
 
 
 
 
 
 
24ef3cd
 
74630d5
 
 
 
 
 
 
 
 
24ef3cd
c4eeba4
 
 
 
 
24ef3cd
 
 
 
 
 
 
 
 
 
 
 
 
 
74630d5
 
24ef3cd
 
 
 
 
 
 
 
74630d5
24ef3cd
 
 
 
74630d5
 
 
 
 
24ef3cd
 
 
74630d5
 
 
 
 
 
24ef3cd
 
 
 
74630d5
 
 
 
 
 
 
 
 
 
 
 
24ef3cd
 
 
 
 
74630d5
24ef3cd
 
 
 
 
 
 
 
 
 
 
74630d5
24ef3cd
 
 
 
 
 
74630d5
24ef3cd
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
"""Typed, probabilistic decisions from Qwen3.8-27B in one forward pass (Apple Silicon, MLX).

You pass a STATE (any text) and a dict of QUESTIONS. Each question is one of

    {"type": "choice", "options": [...], "prompt": "..."}   pick one option (up to 16)
    {"type": "score",  "levels":  [...], "prompt": "..."}   ordered levels; also returns the expected level index
    {"type": "noul",   "statement": "..."}                  probability that the statement is true

and get back, per question, the chosen value, a probability for every option and a confidence.
Answers can only be one of the options you supplied: nothing is generated or parsed.

How it works
  * the model is Qwen3.8-27B truncated to its first 56 layers (of 64)
  * the questions go first and are prefilled once per schema, then cached; each call only
    processes the state plus one short trigger per question, in a single forward pass
  * a small label-free readout head (trained by self-distillation from the full 64-layer model)
    maps the layer-56 hidden state at each trigger to the option probabilities
  * each call is capped at token_budget tokens (default 160, which keeps a call under ~500 ms on an
    M5 Pro); longer states are cut in the middle (first 40% and last 60% of their tokens are kept)
  * "probs" maps option text to probability; if two options share the same text use "probs_list"
  * image=... (PNG/JPEG bytes, base64 str, PIL image or uint8 array) adds one picture to the state; it is
    scaled to image_max_side on its long side (default 576, about 180 image tokens for a 16:9 screenshot)
    and is never truncated (token_budget applies to the text only); needs vision_common.py and
    vision_mlx.py next to this file

    from decider import Decider
    d = Decider("dowands/neojev-mlx")      # or a local folder
    d.ask("Customer says the refund for order 4417 never arrived and they will dispute the charge.",
          {"queue": {"type": "choice", "options": ["Billing", "Technical", "Sales"],
                     "prompt": "Which team should handle this?"},
           "urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
           "churn": {"type": "noul", "statement": "The customer is likely to leave."}})
    d.ask("", {"screen": {"type": "choice", "options": ["main menu", "map", "settings"]}},
          image=open("screenshot.png", "rb").read())
"""
import json
import os
from collections import OrderedDict

import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_lm import load
from mlx_lm.models.base import create_attention_mask, create_ssm_mask
from mlx_lm.models.cache import ArraysCache, KVCache

LETTERS = "ABCDEFGHIJKLMNOP"
VOCAB = list(LETTERS) + ["Yes", "No"]
SYSTEM = ("You are a decision engine. You get a list of QUESTIONS and a STATE. For each question, "
          "answer with exactly one option label.")


def _block(i, name, spec):
    t = spec["type"]
    if t == "noul":
        return f"Q{i} ({name}): Is this statement true? \"{spec['statement']}\"\nOPTIONS: Yes / No", ["Yes", "No"], ["Yes", "No"]
    opts = spec["options"] if t == "choice" else spec["levels"]
    if len(opts) > len(LETTERS):
        raise ValueError(f"question {name!r}: at most {len(LETTERS)} options are supported")
    labels = list(LETTERS[: len(opts)])
    lines = "\n".join(f"{l}. {o}" for l, o in zip(labels, opts))
    return f"Q{i} ({name}): {spec.get('prompt', 'Which option fits best?')}\nOPTIONS:\n{lines}", labels, opts


def _copy(cache):
    out = []
    for c in cache:
        if isinstance(c, KVCache):
            t = KVCache()
            if c.keys is not None:
                t.keys, t.values, t.offset = c.keys[..., : c.offset, :], c.values[..., : c.offset, :], c.offset
        else:
            t = ArraysCache(len(c.cache))
            t.cache = list(c.cache)
        out.append(t)
    return out


class Decider:
    def __init__(self, path="dowands/neojev-mlx", token_budget=160):
        if not os.path.isdir(path):  # a Hugging Face repo id: fetch weights, head and config once
            from huggingface_hub import snapshot_download
            path = snapshot_download(path)
        self.model, self.tok = load(path)
        self.lm = self.model.language_model
        self.budget = token_budget
        cfg = json.load(open(os.path.join(path, "config.json")))
        self.vision = self.mrope = None
        if cfg.get("vision_config"):
            from vision_mlx import install_mrope, load_vision
            self.vision = load_vision(path)
            self.mrope = install_mrope(self.lm)
            self.image_token_id = cfg["image_token_id"]
        t = self.tok.apply_chat_template(
            [{"role": "system", "content": SYSTEM}, {"role": "user", "content": "<<U>>"}],
            tokenize=False, add_generation_prompt=True, enable_thinking=False)
        self.head, self.tail = t.split("<<U>>")
        # text calls read out through decider_head.npz; image calls through decider_head_vision.npz when present
        self.text_head = self._load_head(os.path.join(path, "decider_head.npz"))
        vision = os.path.join(path, "decider_head_vision.npz")
        self.vision_head = self._load_head(vision) if os.path.exists(vision) else self.text_head
        ids = []
        for l in VOCAB:
            e = self.tok.encode(" " + l, add_special_tokens=False)
            ids.append(e[0] if len(e) == 1 else self.tok.encode(l, add_special_tokens=False)[0])
        head = self.lm.lm_head
        ix = mx.array(ids)
        if isinstance(head, nn.QuantizedLinear):
            self.W = mx.dequantize(head.weight[ix], head.scales[ix], head.biases[ix], head.group_size, head.bits)
        else:
            self.W = head.weight[ix]
        self.W = self.W.astype(mx.float32)
        self.norm_w = self.lm.model.norm.weight.astype(mx.float32)
        self.eps = self.lm.model.norm.eps
        self._schema_cache = OrderedDict()   # LRU of cached question prefixes
        self.max_schemas = 64

    @staticmethod
    def _load_head(f):
        z = np.load(f)
        return {"U": mx.array(z["U"]), "V": mx.array(z["V"]), "b": mx.array(z["b"]), "c": mx.array(z["c"]),
                "T": float(z["T"]), "n_pos": int(z["n_pos"])}

    def _enc(self, s):
        return self.tok.encode(s, add_special_tokens=False)

    def _run(self, ids, cache, image=None):
        """image: (embeddings [n, d], grid_thw) for the image-pad tokens in ids, or None."""
        m = self.lm.model
        h = m.embed_tokens(mx.array([ids]))
        fa = create_attention_mask(h, cache[m.fa_idx])
        ssm = create_ssm_mask(h, cache[m.ssm_idx])
        if image is not None:
            from vision_common import mrope_positions
            emb, grid = image
            where = np.nonzero(np.array(ids) == self.image_token_id)[0]
            h[0, mx.array(where)] = emb.astype(h.dtype)
            self.mrope.positions = mx.array(mrope_positions(ids, cache[m.fa_idx].offset, grid, self.image_token_id))
        try:
            for layer, c in zip(m.layers, cache):
                h = layer(h, mask=ssm if layer.is_linear else fa, cache=c)
        finally:
            if self.mrope is not None:
                self.mrope.positions = None
        return h[0]  # pre-norm residual stream of the last layer

    def _image(self, image, max_side):
        from vision_common import MAX_SIDE_LIMIT, MIN_SIDE, image_block, prepare_image
        if self.vision is None:
            raise ValueError("this model has no vision tower (use a release folder with vision weights)")
        if not MIN_SIDE <= int(max_side) <= MAX_SIDE_LIMIT:
            raise ValueError(f"image_max_side must be between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
        patches, grid = prepare_image(image, int(max_side))
        return image_block(self.tok, grid), (self.vision(mx.array(patches), grid), grid)

    def _prefix(self, key, text):
        if key in self._schema_cache:
            self._schema_cache.move_to_end(key)
        else:
            while len(self._schema_cache) >= self.max_schemas:
                self._schema_cache.popitem(last=False)
            cache = self.lm.make_cache()
            self._run(self._enc(text), cache)
            mx.eval([c.state for c in cache])
            self._schema_cache[key] = cache
        return self._schema_cache[key]

    def _cap(self, state, reserved):
        limit = max(32, self.budget - reserved)
        ids = self._enc(state)
        if len(ids) <= limit:
            return state
        h = int(limit * 0.4)
        return self.tok.decode(ids[:h]) + " ... " + self.tok.decode(ids[-(limit - h):])

    def hidden(self, state, questions, cache_schema=True, image=None, image_max_side=576):
        """Layer-56 residual at each trigger (used by ask(), and to fit the readout head)."""
        blocks, meta = [], []
        for i, (name, spec) in enumerate(questions.items(), 1):
            b, labels, opts = _block(i, name, spec)
            blocks.append(b)
            meta.append((i, name, spec, labels, opts))
        qtext = "QUESTIONS:\n" + "\n\n".join(blocks)
        trig_n = sum(len(self._enc(("" if k == 0 else "\n") + f"Q{i}:")) for k, (i, *_ ) in enumerate(meta))
        reserved = len(self._enc(self.tail)) + trig_n + 4
        img_ids, img = self._image(image, image_max_side) if image is not None else ([], None)
        if cache_schema:
            # key keeps question order: the Q1/Q2... numbering in the cached prefix depends on it
            base = self._prefix(json.dumps(list(questions.items()), sort_keys=True), self.head + qtext + "\n\nSTATE:\n")
            body = self._cap(state, reserved)
            if img is None:
                seq = self._enc(body + self.tail)
            else:
                from vision_common import noul_reminder
                seq = img_ids + self._enc(("\n" + body if body else "") + noul_reminder(questions) + self.tail)
        else:
            base = self._prefix("__head__", self.head)
            reserved += len(self._enc("STATE:\n\n\n" + qtext))
            if img is None:
                seq = self._enc("STATE:\n" + self._cap(state, reserved) + "\n\n" + qtext + self.tail)
            else:
                text = self._cap(state, reserved)
                seq = (self._enc("STATE:\n") + img_ids
                       + self._enc(("\n" + text if text else "") + "\n\n" + qtext + self.tail))
        pos = []
        for k, (i, *_ ) in enumerate(meta):
            seq += self._enc(("" if k == 0 else "\n") + f"Q{i}:")
            pos.append(len(seq) - 1)
        h = self._run(seq, _copy(base), img)[mx.array(pos)].astype(mx.float32)
        return h, meta, len(seq)

    def ask(self, state, questions, cache_schema=True, image=None, image_max_side=576):
        """cache_schema=True: questions first, cached across calls with the same schema (use for a fixed
        set of questions asked about many states). False: state first, question after, nothing cached
        (use for one-off questions whose options change every call, e.g. multiple-choice QA).
        image: optional picture that becomes part of the state (see the module docstring)."""
        h, meta, n = self.hidden(state, questions, cache_schema, image, image_max_side)
        hd = self.vision_head if image is not None else self.text_head
        pb = hd["c"][mx.array(np.minimum(np.arange(len(meta)), hd["n_pos"] - 1))]
        x = h + (h @ hd["U"]) @ hd["V"] + hd["b"] + pb
        x = x * mx.rsqrt(mx.mean(x * x, axis=-1, keepdims=True) + self.eps) * self.norm_w
        logits = x @ self.W.T
        mx.eval(logits)
        res = {}
        for k, (i, name, spec, labels, opts) in enumerate(meta):
            z = np.array(logits[k][mx.array([VOCAB.index(l) for l in labels])]) / hd["T"]
            p = np.exp(z - np.logaddexp.reduce(z))
            j = int(p.argmax())
            r = {"probs": {str(o): float(v) for o, v in zip(opts, p)}, "probs_list": [float(v) for v in p],
                 "confidence": float(p[j])}
            if spec["type"] == "noul":
                r["value"] = float(p[0])
            else:
                r["value"] = opts[j]
            if spec["type"] == "score":
                r["expected_index"] = float((p * np.arange(len(p))).sum())
            res[name] = r
        res["_tokens"] = n
        return res


if __name__ == "__main__":
    import sys
    import time
    d = Decider(sys.argv[1] if len(sys.argv) > 1 else "dowands/neojev-mlx")
    qs = {"queue": {"type": "choice", "options": ["Billing and Payments", "Technical Support", "Sales"],
                    "prompt": "Which team should handle this ticket?"},
          "urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
          "churn": {"type": "noul", "statement": "The customer is likely to leave."}}
    s = "Customer emailed twice about a failed refund for order #4417 and says they will dispute the charge tomorrow."
    d.ask(s, qs)
    t = time.perf_counter()
    out = d.ask(s, qs)
    print(json.dumps(out, indent=1))
    print(f"{(time.perf_counter() - t) * 1000:.0f} ms")