neojev / decider_cuda.py
dowands's picture
Image input: vision head, vision_common.py, decider_cuda/server updates, card
946363b verified
Raw History Blame Contribute Delete
13.6 kB
"""NeoJev on NVIDIA GPUs (PyTorch + transformers). Same API and method as decider.py (MLX).
Weights: dowands/neojev (the official Qwen/Qwen3.8-27B-FP8 checkpoint cut to its first 56 decoder
layers; the full Qwen/Qwen3.8-27B-FP8 also works, only its first 56 layers are loaded). Needs a GPU with FP8 support for the FP8 checkpoint (Ada / Hopper or newer) and about 30 GB of memory.
Install flash-linear-attention and causal-conv1d for the fast Gated DeltaNet kernels; without them
transformers falls back to a much slower pure-PyTorch path.
from decider_cuda import Decider
d = Decider("dowands/neojev")
d.ask("Customer says the refund never arrived.",
{"queue": {"type": "choice", "options": ["Billing", "Technical"], "prompt": "Which team?"}})
d.ask("", {"screen": {"type": "choice", "options": ["main menu", "map", "settings"]}},
image=open("screenshot.png", "rb").read()) # needs vision_common.py next to this file
"""
import copy
import json
import os
from collections import OrderedDict
import numpy as np
import torch
from transformers import AutoConfig, AutoModelForImageTextToText, AutoTokenizer
from transformers.cache_utils import DynamicCache
LETTERS = "ABCDEFGHIJKLMNOP"
VOCAB = list(LETTERS) + ["Yes", "No"]
SYSTEM = ("You are a decision engine. You get a list of QUESTIONS and a STATE. For each question, "
"answer with exactly one option label.")
def _block(i, name, spec):
t = spec["type"]
if t == "noul":
return f"Q{i} ({name}): Is this statement true? \"{spec['statement']}\"\nOPTIONS: Yes / No", ["Yes", "No"], ["Yes", "No"]
opts = spec["options"] if t == "choice" else spec["levels"]
if len(opts) > len(LETTERS):
raise ValueError(f"question {name!r}: at most {len(LETTERS)} options are supported")
labels = list(LETTERS[: len(opts)])
lines = "\n".join(f"{l}. {o}" for l, o in zip(labels, opts))
return f"Q{i} ({name}): {spec.get('prompt', 'Which option fits best?')}\nOPTIONS:\n{lines}", labels, opts
def _dequant_rows(lin, ids):
"""Rows of the LM head as float32, whether the checkpoint stores it in bf16 or block-FP8."""
w = lin.weight
if w.dtype in (torch.float8_e4m3fn, torch.float8_e5m2):
scale = getattr(lin, "weight_scale_inv", None)
rows = w[ids].to(torch.float32)
if scale is not None: # block-wise scales, typically 128 x 128
bs_r = w.shape[0] // scale.shape[0]
bs_c = w.shape[1] // scale.shape[1]
s = scale.to(torch.float32)[ids // bs_r]
rows = rows * s.repeat_interleave(bs_c, dim=1)[:, : w.shape[1]]
return rows
return w[ids].to(torch.float32)
def _patch_fp8_dense():
"""transformers 5.16 FP8 quantizer looks up an MoE experts implementation that dense models don't set."""
try:
from transformers.integrations.finegrained_fp8 import FP8Experts
FP8Experts._impl_tp_layer_overrides.setdefault(None, {})
except Exception:
pass
class Decider:
def __init__(self, model_path="dowands/neojev", head=None, n_layers=56, token_budget=160,
device="cuda", vision_head=None):
_patch_fp8_dense()
if head is None: # the readout head ships next to the weights (local folder or Hugging Face repo)
local = os.path.join(model_path, "decider_head.npz")
if os.path.exists(local):
head = local
else:
from huggingface_hub import hf_hub_download
head = hf_hub_download(model_path, "decider_head.npz")
if vision_head is None: # optional second head for image calls, next to the first one
local = os.path.join(model_path, "decider_head_vision.npz")
if os.path.exists(local):
vision_head = local
elif not os.path.isdir(model_path):
try:
from huggingface_hub import hf_hub_download
vision_head = hf_hub_download(model_path, "decider_head_vision.npz")
except Exception:
vision_head = None
cfg = AutoConfig.from_pretrained(model_path)
qc = getattr(cfg, "quantization_config", None)
if isinstance(qc, dict) and qc.get("modules_to_not_convert"):
# the FP8 checkpoint lists MoE router names ("...mlp.gate") that prefix-match the dense "mlp.gate_proj",
# which would then load raw FP8 bytes as bf16 and drop their scales; keep gate_proj quantized
qc["modules_to_not_convert"] = [m for m in qc["modules_to_not_convert"] if not m.endswith(".mlp.gate")]
tc = cfg.text_config
tc.num_hidden_layers = n_layers
tc.layer_types = tc.layer_types[:n_layers]
self.model = AutoModelForImageTextToText.from_pretrained(model_path, config=cfg, dtype="auto",
device_map=device)
self.model.eval()
self.tok = AutoTokenizer.from_pretrained(model_path)
self.image_token_id = getattr(cfg, "image_token_id", None)
self.text = self.model.model.language_model
self.final_norm = self.text.norm
self.text.norm = torch.nn.Identity() # we read the pre-norm residual stream of the last kept layer
self.device = device
self.budget = token_budget
t = self.tok.apply_chat_template(
[{"role": "system", "content": SYSTEM}, {"role": "user", "content": "<<U>>"}],
tokenize=False, add_generation_prompt=True, enable_thinking=False)
self.head, self.tail = t.split("<<U>>")
# text calls read out through `head`; image calls through `vision_head` when there is one
self.text_head = self._load_head(head)
self.vision_head = self._load_head(vision_head) if vision_head else self.text_head
ids = []
for l in VOCAB:
e = self.tok.encode(" " + l, add_special_tokens=False)
ids.append(e[0] if len(e) == 1 else self.tok.encode(l, add_special_tokens=False)[0])
self.W = _dequant_rows(self.model.lm_head, torch.tensor(ids, device=device))
# Qwen3.5 RMSNorm in transformers computes x_hat * (1 + weight): the checkpoint stores weight - 1
self.norm_w = 1.0 + self.final_norm.weight.to(torch.float32)
self.eps = self.final_norm.variance_epsilon if hasattr(self.final_norm, "variance_epsilon") else self.final_norm.eps
self._schema_cache = OrderedDict() # LRU of cached question prefixes
self.max_schemas = 64
def _load_head(self, f):
z = np.load(f)
t = lambda k: torch.tensor(z[k], dtype=torch.float32, device=self.device)
return {"U": t("U"), "V": t("V"), "b": t("b"), "c": t("c"), "T": float(z["T"]), "n_pos": int(z["n_pos"])}
def _enc(self, s):
return self.tok.encode(s, add_special_tokens=False)
@torch.inference_mode()
def _run(self, ids, cache, image=None):
"""image: (embeddings [n, d], grid_thw) for the image-pad tokens in ids, or None."""
if image is None:
out = self.text(input_ids=torch.tensor([ids], device=self.device), past_key_values=cache, use_cache=True)
return out.last_hidden_state[0]
from vision_common import mrope_positions
emb, grid = image
x = torch.tensor([ids], device=self.device)
h = self.text.embed_tokens(x)
mask = (x == self.image_token_id)[..., None].expand_as(h)
h = h.masked_scatter(mask, emb.to(h.dtype))
pos = torch.tensor(mrope_positions(ids, cache.get_seq_length(), grid, self.image_token_id),
device=self.device)[:, None, :]
out = self.text(inputs_embeds=h, position_ids=pos, past_key_values=cache, use_cache=True)
return out.last_hidden_state[0]
@torch.inference_mode()
def _image(self, image, max_side):
from vision_common import MAX_SIDE_LIMIT, MIN_SIDE, image_block, prepare_image
if getattr(self.model.model, "visual", None) is None or self.image_token_id is None:
raise ValueError("this model has no vision tower")
if not MIN_SIDE <= int(max_side) <= MAX_SIDE_LIMIT:
raise ValueError(f"image_max_side must be between {MIN_SIDE} and {MAX_SIDE_LIMIT}")
patches, grid = prepare_image(image, int(max_side))
out = self.model.model.get_image_features(torch.tensor(patches, device=self.device),
torch.tensor([grid], device=self.device))
emb = out.pooler_output if hasattr(out, "pooler_output") else out
if isinstance(emb, (list, tuple)):
emb = torch.cat(list(emb), dim=0)
return image_block(self.tok, grid), (emb, grid)
def _prefix(self, key, text):
if key in self._schema_cache:
self._schema_cache.move_to_end(key)
else:
while len(self._schema_cache) >= self.max_schemas:
self._schema_cache.popitem(last=False)
cache = DynamicCache(config=self.text.config)
self._run(self._enc(text), cache)
self._schema_cache[key] = cache
return self._schema_cache[key]
def _cap(self, state, reserved):
limit = max(32, self.budget - reserved)
ids = self._enc(state)
if len(ids) <= limit:
return state
h = int(limit * 0.4)
return self.tok.decode(ids[:h]) + " ... " + self.tok.decode(ids[-(limit - h):])
@torch.inference_mode()
def hidden(self, state, questions, cache_schema=True, image=None, image_max_side=576):
"""Layer-56 residual at each trigger (used by ask(), and to re-fit the readout head on this backend)."""
blocks, meta = [], []
for i, (name, spec) in enumerate(questions.items(), 1):
b, labels, opts = _block(i, name, spec)
blocks.append(b)
meta.append((i, name, spec, labels, opts))
qtext = "QUESTIONS:\n" + "\n\n".join(blocks)
trig_n = sum(len(self._enc(("" if k == 0 else "\n") + f"Q{i}:")) for k, (i, *_ ) in enumerate(meta))
reserved = len(self._enc(self.tail)) + trig_n + 4
img_ids, img = self._image(image, image_max_side) if image is not None else ([], None)
if cache_schema:
base = self._prefix(json.dumps(list(questions.items()), sort_keys=True), self.head + qtext + "\n\nSTATE:\n")
body = self._cap(state, reserved)
if img is None:
seq = self._enc(body + self.tail)
else:
from vision_common import noul_reminder
seq = img_ids + self._enc(("\n" + body if body else "") + noul_reminder(questions) + self.tail)
else:
base = self._prefix("__head__", self.head)
reserved += len(self._enc("STATE:\n\n\n" + qtext))
if img is None:
seq = self._enc("STATE:\n" + self._cap(state, reserved) + "\n\n" + qtext + self.tail)
else:
text = self._cap(state, reserved)
seq = (self._enc("STATE:\n") + img_ids
+ self._enc(("\n" + text if text else "") + "\n\n" + qtext + self.tail))
pos = []
for k, (i, *_ ) in enumerate(meta):
seq += self._enc(("" if k == 0 else "\n") + f"Q{i}:")
pos.append(len(seq) - 1)
h = self._run(seq, copy.deepcopy(base), img)[pos].to(torch.float32)
return h, meta, len(seq)
@torch.inference_mode()
def ask(self, state, questions, cache_schema=True, image=None, image_max_side=576):
h, meta, n = self.hidden(state, questions, cache_schema, image, image_max_side)
hd = self.vision_head if image is not None else self.text_head
pb = hd["c"][torch.clamp(torch.arange(len(meta), device=self.device), max=hd["n_pos"] - 1)]
x = h + (h @ hd["U"]) @ hd["V"] + hd["b"] + pb
x = x * torch.rsqrt((x * x).mean(-1, keepdim=True) + self.eps) * self.norm_w
logits = (x @ self.W.T).cpu().numpy()
res = {}
for k, (i, name, spec, labels, opts) in enumerate(meta):
z = logits[k][[VOCAB.index(l) for l in labels]] / hd["T"]
p = np.exp(z - np.logaddexp.reduce(z))
j = int(p.argmax())
r = {"probs": {str(o): float(v) for o, v in zip(opts, p)}, "probs_list": [float(v) for v in p],
"confidence": float(p[j])}
r["value"] = float(p[0]) if spec["type"] == "noul" else opts[j]
if spec["type"] == "score":
r["expected_index"] = float((p * np.arange(len(p))).sum())
res[name] = r
res["_tokens"] = n
return res
if __name__ == "__main__":
import sys
import time
d = Decider(sys.argv[1] if len(sys.argv) > 1 else "dowands/neojev",
head=sys.argv[2] if len(sys.argv) > 2 else None)
qs = {"queue": {"type": "choice", "options": ["Billing and Payments", "Technical Support", "Sales"],
"prompt": "Which team should handle this ticket?"},
"urgency": {"type": "score", "levels": ["low", "medium", "high"], "prompt": "How urgent is it?"},
"churn": {"type": "noul", "statement": "The customer is likely to leave."}}
s = "Customer emailed twice about a failed refund for order #4417 and says they will dispute the charge tomorrow."
d.ask(s, qs)
torch.cuda.synchronize()
t = time.perf_counter()
out = d.ask(s, qs)
torch.cuda.synchronize()
print(json.dumps(out, indent=1))
print(f"{(time.perf_counter() - t) * 1000:.0f} ms")