neojev-mlx / decider.py
dowands's picture
Add files using upload-large-folder tool
74630d5 verified
Raw History Blame Contribute Delete
13.1 kB
"""Typed, probabilistic decisions from Qwen3.8-27B in one forward pass (Apple Silicon, MLX).
You pass a STATE (any text) and a dict of QUESTIONS. Each question is one of
{"type": "choice", "options": [...], "prompt": "..."} pick one option (up to 16)
{"type": "score", "levels": [...], "prompt": "..."} ordered levels; also returns the expected level index
{"type": "noul", "statement": "..."} probability that the statement is true
and get back, per question, the chosen value, a probability for every option and a confidence.
Answers can only be one of the options you supplied: nothing is generated or parsed.
How it works
* the model is Qwen3.8-27B truncated to its first 56 layers (of 64)
* the questions go first and are prefilled once per schema, then cached; each call only
processes the state plus one short trigger per question, in a single forward pass
* a small label-free readout head (trained by self-distillation from the full 64-layer model)
maps the layer-56 hidden state at each trigger to the option probabilities
* each call is capped at token_budget tokens (default 160, which keeps a call under ~500 ms on an
M5 Pro); longer states are cut in the middle (first 40% and last 60% of their tokens are kept)
* "probs" maps option text to probability; if two options share the same text use "probs_list"
* image=... (PNG/JPEG bytes, base64 str, PIL image or uint8 array) adds one picture to the state; it is
scaled to image_max_side on its long side (default 576, about 180 image tokens for a 16:9 screenshot)
and is never truncated (token_budget applies to the text only); needs vision_common.py and
vision_mlx.py next to this file
from decider import Decider
d = Decider("dowands/neojev-mlx") # or a local folder
d.ask("Customer says the refund for order 4417 never arrived and they will dispute the charge.",
{"queue": {"type": "choice", "options": ["Billing", "Technical", "Sales"],
"prompt": "Which team should handle this?"},
"urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
"churn": {"type": "noul", "statement": "The customer is likely to leave."}})
d.ask("", {"screen": {"type": "choice", "options": ["main menu", "map", "settings"]}},
image=open("screenshot.png", "rb").read())
"""
import json
import os
from collections import OrderedDict
import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_lm import load
from mlx_lm.models.base import create_attention_mask, create_ssm_mask
from mlx_lm.models.cache import ArraysCache, KVCache
LETTERS = "ABCDEFGHIJKLMNOP"
VOCAB = list(LETTERS) + ["Yes", "No"]
SYSTEM = ("You are a decision engine. You get a list of QUESTIONS and a STATE. For each question, "
"answer with exactly one option label.")
def _block(i, name, spec):
t = spec["type"]
if t == "noul":
return f"Q{i} ({name}): Is this statement true? \"{spec['statement']}\"\nOPTIONS: Yes / No", ["Yes", "No"], ["Yes", "No"]
opts = spec["options"] if t == "choice" else spec["levels"]
if len(opts) > len(LETTERS):
raise ValueError(f"question {name!r}: at most {len(LETTERS)} options are supported")
labels = list(LETTERS[: len(opts)])
lines = "\n".join(f"{l}. {o}" for l, o in zip(labels, opts))
return f"Q{i} ({name}): {spec.get('prompt', 'Which option fits best?')}\nOPTIONS:\n{lines}", labels, opts
def _copy(cache):
out = []
for c in cache:
if isinstance(c, KVCache):
t = KVCache()
if c.keys is not None:
t.keys, t.values, t.offset = c.keys[..., : c.offset, :], c.values[..., : c.offset, :], c.offset
else:
t = ArraysCache(len(c.cache))
t.cache = list(c.cache)
out.append(t)
return out
class Decider:
def __init__(self, path="dowands/neojev-mlx", token_budget=160):
if not os.path.isdir(path): # a Hugging Face repo id: fetch weights, head and config once
from huggingface_hub import snapshot_download
path = snapshot_download(path)
self.model, self.tok = load(path)
self.lm = self.model.language_model
self.budget = token_budget
cfg = json.load(open(os.path.join(path, "config.json")))
self.vision = self.mrope = None
if cfg.get("vision_config"):
from vision_mlx import install_mrope, load_vision
self.vision = load_vision(path)
self.mrope = install_mrope(self.lm)
self.image_token_id = cfg["image_token_id"]
t = self.tok.apply_chat_template(
[{"role": "system", "content": SYSTEM}, {"role": "user", "content": "<<U>>"}],
tokenize=False, add_generation_prompt=True, enable_thinking=False)
self.head, self.tail = t.split("<<U>>")
# text calls read out through decider_head.npz; image calls through decider_head_vision.npz when present
self.text_head = self._load_head(os.path.join(path, "decider_head.npz"))
vision = os.path.join(path, "decider_head_vision.npz")
self.vision_head = self._load_head(vision) if os.path.exists(vision) else self.text_head
ids = []
for l in VOCAB:
e = self.tok.encode(" " + l, add_special_tokens=False)
ids.append(e[0] if len(e) == 1 else self.tok.encode(l, add_special_tokens=False)[0])
head = self.lm.lm_head
ix = mx.array(ids)
if isinstance(head, nn.QuantizedLinear):
self.W = mx.dequantize(head.weight[ix], head.scales[ix], head.biases[ix], head.group_size, head.bits)
else:
self.W = head.weight[ix]
self.W = self.W.astype(mx.float32)
self.norm_w = self.lm.model.norm.weight.astype(mx.float32)
self.eps = self.lm.model.norm.eps
self._schema_cache = OrderedDict() # LRU of cached question prefixes
self.max_schemas = 64
@staticmethod
def _load_head(f):
z = np.load(f)
return {"U": mx.array(z["U"]), "V": mx.array(z["V"]), "b": mx.array(z["b"]), "c": mx.array(z["c"]),
"T": float(z["T"]), "n_pos": int(z["n_pos"])}
def _enc(self, s):
return self.tok.encode(s, add_special_tokens=False)
def _run(self, ids, cache, image=None):
"""image: (embeddings [n, d], grid_thw) for the image-pad tokens in ids, or None."""
m = self.lm.model
h = m.embed_tokens(mx.array([ids]))
fa = create_attention_mask(h, cache[m.fa_idx])
ssm = create_ssm_mask(h, cache[m.ssm_idx])
if image is not None:
from vision_common import mrope_positions
emb, grid = image
where = np.nonzero(np.array(ids) == self.image_token_id)[0]
h[0, mx.array(where)] = emb.astype(h.dtype)
self.mrope.positions = mx.array(mrope_positions(ids, cache[m.fa_idx].offset, grid, self.image_token_id))
try:
for layer, c in zip(m.layers, cache):
h = layer(h, mask=ssm if layer.is_linear else fa, cache=c)
finally:
if self.mrope is not None:
self.mrope.positions = None
return h[0] # pre-norm residual stream of the last layer
def _image(self, image, max_side):
from vision_common import MAX_SIDE_LIMIT, MIN_SIDE, image_block, prepare_image
if self.vision is None:
raise ValueError("this model has no vision tower (use a release folder with vision weights)")
if not MIN_SIDE <= int(max_side) <= MAX_SIDE_LIMIT:
raise ValueError(f"image_max_side must be between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
patches, grid = prepare_image(image, int(max_side))
return image_block(self.tok, grid), (self.vision(mx.array(patches), grid), grid)
def _prefix(self, key, text):
if key in self._schema_cache:
self._schema_cache.move_to_end(key)
else:
while len(self._schema_cache) >= self.max_schemas:
self._schema_cache.popitem(last=False)
cache = self.lm.make_cache()
self._run(self._enc(text), cache)
mx.eval([c.state for c in cache])
self._schema_cache[key] = cache
return self._schema_cache[key]
def _cap(self, state, reserved):
limit = max(32, self.budget - reserved)
ids = self._enc(state)
if len(ids) <= limit:
return state
h = int(limit * 0.4)
return self.tok.decode(ids[:h]) + " ... " + self.tok.decode(ids[-(limit - h):])
def hidden(self, state, questions, cache_schema=True, image=None, image_max_side=576):
"""Layer-56 residual at each trigger (used by ask(), and to fit the readout head)."""
blocks, meta = [], []
for i, (name, spec) in enumerate(questions.items(), 1):
b, labels, opts = _block(i, name, spec)
blocks.append(b)
meta.append((i, name, spec, labels, opts))
qtext = "QUESTIONS:\n" + "\n\n".join(blocks)
trig_n = sum(len(self._enc(("" if k == 0 else "\n") + f"Q{i}:")) for k, (i, *_ ) in enumerate(meta))
reserved = len(self._enc(self.tail)) + trig_n + 4
img_ids, img = self._image(image, image_max_side) if image is not None else ([], None)
if cache_schema:
# key keeps question order: the Q1/Q2... numbering in the cached prefix depends on it
base = self._prefix(json.dumps(list(questions.items()), sort_keys=True), self.head + qtext + "\n\nSTATE:\n")
body = self._cap(state, reserved)
if img is None:
seq = self._enc(body + self.tail)
else:
from vision_common import noul_reminder
seq = img_ids + self._enc(("\n" + body if body else "") + noul_reminder(questions) + self.tail)
else:
base = self._prefix("__head__", self.head)
reserved += len(self._enc("STATE:\n\n\n" + qtext))
if img is None:
seq = self._enc("STATE:\n" + self._cap(state, reserved) + "\n\n" + qtext + self.tail)
else:
text = self._cap(state, reserved)
seq = (self._enc("STATE:\n") + img_ids
+ self._enc(("\n" + text if text else "") + "\n\n" + qtext + self.tail))
pos = []
for k, (i, *_ ) in enumerate(meta):
seq += self._enc(("" if k == 0 else "\n") + f"Q{i}:")
pos.append(len(seq) - 1)
h = self._run(seq, _copy(base), img)[mx.array(pos)].astype(mx.float32)
return h, meta, len(seq)
def ask(self, state, questions, cache_schema=True, image=None, image_max_side=576):
"""cache_schema=True: questions first, cached across calls with the same schema (use for a fixed
set of questions asked about many states). False: state first, question after, nothing cached
(use for one-off questions whose options change every call, e.g. multiple-choice QA).
image: optional picture that becomes part of the state (see the module docstring)."""
h, meta, n = self.hidden(state, questions, cache_schema, image, image_max_side)
hd = self.vision_head if image is not None else self.text_head
pb = hd["c"][mx.array(np.minimum(np.arange(len(meta)), hd["n_pos"] - 1))]
x = h + (h @ hd["U"]) @ hd["V"] + hd["b"] + pb
x = x * mx.rsqrt(mx.mean(x * x, axis=-1, keepdims=True) + self.eps) * self.norm_w
logits = x @ self.W.T
mx.eval(logits)
res = {}
for k, (i, name, spec, labels, opts) in enumerate(meta):
z = np.array(logits[k][mx.array([VOCAB.index(l) for l in labels])]) / hd["T"]
p = np.exp(z - np.logaddexp.reduce(z))
j = int(p.argmax())
r = {"probs": {str(o): float(v) for o, v in zip(opts, p)}, "probs_list": [float(v) for v in p],
"confidence": float(p[j])}
if spec["type"] == "noul":
r["value"] = float(p[0])
else:
r["value"] = opts[j]
if spec["type"] == "score":
r["expected_index"] = float((p * np.arange(len(p))).sum())
res[name] = r
res["_tokens"] = n
return res
if __name__ == "__main__":
import sys
import time
d = Decider(sys.argv[1] if len(sys.argv) > 1 else "dowands/neojev-mlx")
qs = {"queue": {"type": "choice", "options": ["Billing and Payments", "Technical Support", "Sales"],
"prompt": "Which team should handle this ticket?"},
"urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
"churn": {"type": "noul", "statement": "The customer is likely to leave."}}
s = "Customer emailed twice about a failed refund for order #4417 and says they will dispute the charge tomorrow."
d.ask(s, qs)
t = time.perf_counter()
out = d.ask(s, qs)
print(json.dumps(out, indent=1))
print(f"{(time.perf_counter() - t) * 1000:.0f} ms")