Text Classification
MLX
Safetensors
qwen3_5
neojev
jev
system-one
typed-decisions
classification
calibration
vision
apple-silicon
4-bit precision
Instructions to use dowands/neojev-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use dowands/neojev-mlx with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download dowands/neojev-mlx --local-dir neojev-mlx
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 13,118 Bytes
24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd c4eeba4 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd c4eeba4 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd c4eeba4 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd 74630d5 24ef3cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 | """Typed, probabilistic decisions from Qwen3.8-27B in one forward pass (Apple Silicon, MLX).
You pass a STATE (any text) and a dict of QUESTIONS. Each question is one of
{"type": "choice", "options": [...], "prompt": "..."} pick one option (up to 16)
{"type": "score", "levels": [...], "prompt": "..."} ordered levels; also returns the expected level index
{"type": "noul", "statement": "..."} probability that the statement is true
and get back, per question, the chosen value, a probability for every option and a confidence.
Answers can only be one of the options you supplied: nothing is generated or parsed.
How it works
* the model is Qwen3.8-27B truncated to its first 56 layers (of 64)
* the questions go first and are prefilled once per schema, then cached; each call only
processes the state plus one short trigger per question, in a single forward pass
* a small label-free readout head (trained by self-distillation from the full 64-layer model)
maps the layer-56 hidden state at each trigger to the option probabilities
* each call is capped at token_budget tokens (default 160, which keeps a call under ~500 ms on an
M5 Pro); longer states are cut in the middle (first 40% and last 60% of their tokens are kept)
* "probs" maps option text to probability; if two options share the same text use "probs_list"
* image=... (PNG/JPEG bytes, base64 str, PIL image or uint8 array) adds one picture to the state; it is
scaled to image_max_side on its long side (default 576, about 180 image tokens for a 16:9 screenshot)
and is never truncated (token_budget applies to the text only); needs vision_common.py and
vision_mlx.py next to this file
from decider import Decider
d = Decider("dowands/neojev-mlx") # or a local folder
d.ask("Customer says the refund for order 4417 never arrived and they will dispute the charge.",
{"queue": {"type": "choice", "options": ["Billing", "Technical", "Sales"],
"prompt": "Which team should handle this?"},
"urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
"churn": {"type": "noul", "statement": "The customer is likely to leave."}})
d.ask("", {"screen": {"type": "choice", "options": ["main menu", "map", "settings"]}},
image=open("screenshot.png", "rb").read())
"""
import json
import os
from collections import OrderedDict
import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_lm import load
from mlx_lm.models.base import create_attention_mask, create_ssm_mask
from mlx_lm.models.cache import ArraysCache, KVCache
LETTERS = "ABCDEFGHIJKLMNOP"
VOCAB = list(LETTERS) + ["Yes", "No"]
SYSTEM = ("You are a decision engine. You get a list of QUESTIONS and a STATE. For each question, "
"answer with exactly one option label.")
def _block(i, name, spec):
t = spec["type"]
if t == "noul":
return f"Q{i} ({name}): Is this statement true? \"{spec['statement']}\"\nOPTIONS: Yes / No", ["Yes", "No"], ["Yes", "No"]
opts = spec["options"] if t == "choice" else spec["levels"]
if len(opts) > len(LETTERS):
raise ValueError(f"question {name!r}: at most {len(LETTERS)} options are supported")
labels = list(LETTERS[: len(opts)])
lines = "\n".join(f"{l}. {o}" for l, o in zip(labels, opts))
return f"Q{i} ({name}): {spec.get('prompt', 'Which option fits best?')}\nOPTIONS:\n{lines}", labels, opts
def _copy(cache):
out = []
for c in cache:
if isinstance(c, KVCache):
t = KVCache()
if c.keys is not None:
t.keys, t.values, t.offset = c.keys[..., : c.offset, :], c.values[..., : c.offset, :], c.offset
else:
t = ArraysCache(len(c.cache))
t.cache = list(c.cache)
out.append(t)
return out
class Decider:
def __init__(self, path="dowands/neojev-mlx", token_budget=160):
if not os.path.isdir(path): # a Hugging Face repo id: fetch weights, head and config once
from huggingface_hub import snapshot_download
path = snapshot_download(path)
self.model, self.tok = load(path)
self.lm = self.model.language_model
self.budget = token_budget
cfg = json.load(open(os.path.join(path, "config.json")))
self.vision = self.mrope = None
if cfg.get("vision_config"):
from vision_mlx import install_mrope, load_vision
self.vision = load_vision(path)
self.mrope = install_mrope(self.lm)
self.image_token_id = cfg["image_token_id"]
t = self.tok.apply_chat_template(
[{"role": "system", "content": SYSTEM}, {"role": "user", "content": "<<U>>"}],
tokenize=False, add_generation_prompt=True, enable_thinking=False)
self.head, self.tail = t.split("<<U>>")
# text calls read out through decider_head.npz; image calls through decider_head_vision.npz when present
self.text_head = self._load_head(os.path.join(path, "decider_head.npz"))
vision = os.path.join(path, "decider_head_vision.npz")
self.vision_head = self._load_head(vision) if os.path.exists(vision) else self.text_head
ids = []
for l in VOCAB:
e = self.tok.encode(" " + l, add_special_tokens=False)
ids.append(e[0] if len(e) == 1 else self.tok.encode(l, add_special_tokens=False)[0])
head = self.lm.lm_head
ix = mx.array(ids)
if isinstance(head, nn.QuantizedLinear):
self.W = mx.dequantize(head.weight[ix], head.scales[ix], head.biases[ix], head.group_size, head.bits)
else:
self.W = head.weight[ix]
self.W = self.W.astype(mx.float32)
self.norm_w = self.lm.model.norm.weight.astype(mx.float32)
self.eps = self.lm.model.norm.eps
self._schema_cache = OrderedDict() # LRU of cached question prefixes
self.max_schemas = 64
@staticmethod
def _load_head(f):
z = np.load(f)
return {"U": mx.array(z["U"]), "V": mx.array(z["V"]), "b": mx.array(z["b"]), "c": mx.array(z["c"]),
"T": float(z["T"]), "n_pos": int(z["n_pos"])}
def _enc(self, s):
return self.tok.encode(s, add_special_tokens=False)
def _run(self, ids, cache, image=None):
"""image: (embeddings [n, d], grid_thw) for the image-pad tokens in ids, or None."""
m = self.lm.model
h = m.embed_tokens(mx.array([ids]))
fa = create_attention_mask(h, cache[m.fa_idx])
ssm = create_ssm_mask(h, cache[m.ssm_idx])
if image is not None:
from vision_common import mrope_positions
emb, grid = image
where = np.nonzero(np.array(ids) == self.image_token_id)[0]
h[0, mx.array(where)] = emb.astype(h.dtype)
self.mrope.positions = mx.array(mrope_positions(ids, cache[m.fa_idx].offset, grid, self.image_token_id))
try:
for layer, c in zip(m.layers, cache):
h = layer(h, mask=ssm if layer.is_linear else fa, cache=c)
finally:
if self.mrope is not None:
self.mrope.positions = None
return h[0] # pre-norm residual stream of the last layer
def _image(self, image, max_side):
from vision_common import MAX_SIDE_LIMIT, MIN_SIDE, image_block, prepare_image
if self.vision is None:
raise ValueError("this model has no vision tower (use a release folder with vision weights)")
if not MIN_SIDE <= int(max_side) <= MAX_SIDE_LIMIT:
raise ValueError(f"image_max_side must be between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
patches, grid = prepare_image(image, int(max_side))
return image_block(self.tok, grid), (self.vision(mx.array(patches), grid), grid)
def _prefix(self, key, text):
if key in self._schema_cache:
self._schema_cache.move_to_end(key)
else:
while len(self._schema_cache) >= self.max_schemas:
self._schema_cache.popitem(last=False)
cache = self.lm.make_cache()
self._run(self._enc(text), cache)
mx.eval([c.state for c in cache])
self._schema_cache[key] = cache
return self._schema_cache[key]
def _cap(self, state, reserved):
limit = max(32, self.budget - reserved)
ids = self._enc(state)
if len(ids) <= limit:
return state
h = int(limit * 0.4)
return self.tok.decode(ids[:h]) + " ... " + self.tok.decode(ids[-(limit - h):])
def hidden(self, state, questions, cache_schema=True, image=None, image_max_side=576):
"""Layer-56 residual at each trigger (used by ask(), and to fit the readout head)."""
blocks, meta = [], []
for i, (name, spec) in enumerate(questions.items(), 1):
b, labels, opts = _block(i, name, spec)
blocks.append(b)
meta.append((i, name, spec, labels, opts))
qtext = "QUESTIONS:\n" + "\n\n".join(blocks)
trig_n = sum(len(self._enc(("" if k == 0 else "\n") + f"Q{i}:")) for k, (i, *_ ) in enumerate(meta))
reserved = len(self._enc(self.tail)) + trig_n + 4
img_ids, img = self._image(image, image_max_side) if image is not None else ([], None)
if cache_schema:
# key keeps question order: the Q1/Q2... numbering in the cached prefix depends on it
base = self._prefix(json.dumps(list(questions.items()), sort_keys=True), self.head + qtext + "\n\nSTATE:\n")
body = self._cap(state, reserved)
if img is None:
seq = self._enc(body + self.tail)
else:
from vision_common import noul_reminder
seq = img_ids + self._enc(("\n" + body if body else "") + noul_reminder(questions) + self.tail)
else:
base = self._prefix("__head__", self.head)
reserved += len(self._enc("STATE:\n\n\n" + qtext))
if img is None:
seq = self._enc("STATE:\n" + self._cap(state, reserved) + "\n\n" + qtext + self.tail)
else:
text = self._cap(state, reserved)
seq = (self._enc("STATE:\n") + img_ids
+ self._enc(("\n" + text if text else "") + "\n\n" + qtext + self.tail))
pos = []
for k, (i, *_ ) in enumerate(meta):
seq += self._enc(("" if k == 0 else "\n") + f"Q{i}:")
pos.append(len(seq) - 1)
h = self._run(seq, _copy(base), img)[mx.array(pos)].astype(mx.float32)
return h, meta, len(seq)
def ask(self, state, questions, cache_schema=True, image=None, image_max_side=576):
"""cache_schema=True: questions first, cached across calls with the same schema (use for a fixed
set of questions asked about many states). False: state first, question after, nothing cached
(use for one-off questions whose options change every call, e.g. multiple-choice QA).
image: optional picture that becomes part of the state (see the module docstring)."""
h, meta, n = self.hidden(state, questions, cache_schema, image, image_max_side)
hd = self.vision_head if image is not None else self.text_head
pb = hd["c"][mx.array(np.minimum(np.arange(len(meta)), hd["n_pos"] - 1))]
x = h + (h @ hd["U"]) @ hd["V"] + hd["b"] + pb
x = x * mx.rsqrt(mx.mean(x * x, axis=-1, keepdims=True) + self.eps) * self.norm_w
logits = x @ self.W.T
mx.eval(logits)
res = {}
for k, (i, name, spec, labels, opts) in enumerate(meta):
z = np.array(logits[k][mx.array([VOCAB.index(l) for l in labels])]) / hd["T"]
p = np.exp(z - np.logaddexp.reduce(z))
j = int(p.argmax())
r = {"probs": {str(o): float(v) for o, v in zip(opts, p)}, "probs_list": [float(v) for v in p],
"confidence": float(p[j])}
if spec["type"] == "noul":
r["value"] = float(p[0])
else:
r["value"] = opts[j]
if spec["type"] == "score":
r["expected_index"] = float((p * np.arange(len(p))).sum())
res[name] = r
res["_tokens"] = n
return res
if __name__ == "__main__":
import sys
import time
d = Decider(sys.argv[1] if len(sys.argv) > 1 else "dowands/neojev-mlx")
qs = {"queue": {"type": "choice", "options": ["Billing and Payments", "Technical Support", "Sales"],
"prompt": "Which team should handle this ticket?"},
"urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
"churn": {"type": "noul", "statement": "The customer is likely to leave."}}
s = "Customer emailed twice about a failed refund for order #4417 and says they will dispute the charge tomorrow."
d.ask(s, qs)
t = time.perf_counter()
out = d.ask(s, qs)
print(json.dumps(out, indent=1))
print(f"{(time.perf_counter() - t) * 1000:.0f} ms")
|