Text Classification
jev-style
Safetensors
Transformers
qwen3_5
image-text-to-text
decision-model
decision-making
system-one
calibration
classification
long-context
vision
multimodal
qwen3.5
llm-routing
guardrails
Instructions to use chaoliangUNSW/Jev-Style-2B-Decision-v4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- jev-style
How to use chaoliangUNSW/Jev-Style-2B-Decision-v4 with jev-style:
pip install "jev-style[torch]"
from jev_style import JevStyle, noul, choice js = JevStyle.from_pretrained("chaoliangUNSW/Jev-Style-2B-Decision-v4") out = js.decide("I was charged twice for one order.", { "billing": noul("This message is about billing."), "team": choice("Which team should handle it?", ["billing", "shipping", "tech"]), }) print(out["answers"]["team"]["choice"]) - Transformers
How to use chaoliangUNSW/Jev-Style-2B-Decision-v4 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="chaoliangUNSW/Jev-Style-2B-Decision-v4")# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("chaoliangUNSW/Jev-Style-2B-Decision-v4") model = AutoModelForMultimodalLM.from_pretrained("chaoliangUNSW/Jev-Style-2B-Decision-v4", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download src/macjev/runtime/v4_engine.py from chaoliangUNSW/Jev-Style-2B-Decision-v4: direct link, hf CLI and curl.
- Browser
- Download file 18.9 kB
-
https://huggingface.co/chaoliangUNSW/Jev-Style-2B-Decision-v4/resolve/main/src/macjev/runtime/v4_engine.py
- Command line
-
hf download hf://chaoliangUNSW/Jev-Style-2B-Decision-v4/src/macjev/runtime/v4_engine.py
-
curl -L -o v4_engine.py https://huggingface.co/chaoliangUNSW/Jev-Style-2B-Decision-v4/resolve/main/src/macjev/runtime/v4_engine.py
18.9 kB
| """v4 reference scorer: answers ``/v1/systemone`` requests with a ``scripts/finalize_v4.py`` export (``<out>/hf``). | |
| It runs the SAME code as training and finalize -- ``RendererV2.pieces`` -> ``train.evaluate.score_rows`` (block | |
| attention, the vision tower for images) -> softmax(logits / T) with the export's per-qtype temperature -- so its | |
| probabilities equal the finalize parity check's. It is the reference for jev-style 0.6.0 and is what the local | |
| evaluation adapters drive (``macjev.eval.di_v4`` for the Decision Index text + vision suites, ``macjev.serve.v4_server`` | |
| for JevBench and any HTTP client). | |
| * One request = one state + N questions. Each question becomes one row "state + its own question block"; with the | |
| ``sb`` layout a block attends only to the state and itself, so scoring the rows separately equals a shared-prefix | |
| computation (the runtime's prefix reuse is a speed-up, not a different answer). | |
| * Nothing is truncated: a request that does not fit ``total_context_limit`` raises ``InputBudgetError`` | |
| (the DI adapter turns it into ``Unsupported``; the server into HTTP 422 ``input_budget_exceeded``). | |
| * Images (``images=[...]``: paths, data URLs, base64 or bytes) are capped at ``max_pixels`` (default 1,638,400 = | |
| the DI Vision rule: native resolution, at most 1.6 MP) with the kit's LANCZOS downscale, then resized by the same | |
| ``smart_resize`` rule as the training data (multiples of 32 px) and read by the export's own image processor. A | |
| state without ``<|image_k|>`` placeholders gets them added the way the training data writes them | |
| (``{"image": "<|image_1|>", ...}``; several images: ``image_1`` .. ``image_n``). | |
| * Answer arithmetic = ``macjev.serve.engine_adapter``: noul -> p(true); choice -> argmax + probabilities in the | |
| request's option order; score -> expectation over level indices + ``probabilities`` keyed "0".."K-1". | |
| * Speed paths over the same numbers (up to floating-point order; off unless asked for): ``fast_text`` computes a text | |
| request's state once for 2+ questions (``runtime.fast_text``); ``cuda_graphs`` replays one CUDA graph per padded row | |
| length recorded at start-up (``runtime.cuda_graphs``). ``route`` picks the path per request. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import io | |
| import json | |
| import tempfile | |
| import threading | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Sequence | |
| import numpy as np | |
| import torch | |
| from .. import backbone as bb | |
| from ..render import InputBudgetError | |
| from ..render_v2 import VERSION as V2_VERSION, RendererV2 | |
| from ..serve.schema import ParsedRequest, QuestionSpec, parse_request | |
| from ..train.evaluate import score_rows | |
| from ..train.model import pick_device | |
| DI_MAX_PIXELS = 1_638_400 # Decision Index Vision rule: native resolution capped at 1.6 MP per image | |
| ENGINE_VERSION = "macjev-v4-reference-1" | |
| def _image_bytes(ref: Any, root: str | Path | None = None) -> bytes: | |
| """paths (relative to ``root`` first), data URLs, raw base64, bytes, or a PIL image.""" | |
| if hasattr(ref, "save") and hasattr(ref, "size"): | |
| buf = io.BytesIO() | |
| ref.save(buf, format="PNG") | |
| return buf.getvalue() | |
| if isinstance(ref, (bytes, bytearray)): | |
| return bytes(ref) | |
| if not isinstance(ref, str): | |
| raise ValueError(f"unsupported image reference of type {type(ref).__name__}") | |
| import base64 | |
| import binascii | |
| if ref.startswith("data:"): | |
| return base64.b64decode(ref.split(",", 1)[1]) | |
| for p in ([Path(root) / ref] if root else []) + [Path(ref)]: | |
| try: | |
| if p.is_file(): | |
| return p.read_bytes() | |
| except OSError: | |
| pass | |
| try: | |
| return base64.b64decode(ref, validate=True) | |
| except (binascii.Error, ValueError): | |
| raise FileNotFoundError(f"image not found: {ref[:80]}") from None | |
| def _cap(raw: bytes, max_pixels: int | None): | |
| """PIL image capped at ``max_pixels`` (aspect kept, LANCZOS -- the DI kit's ``vision.images.capped`` rule).""" | |
| from PIL import Image | |
| im = Image.open(io.BytesIO(raw)) | |
| im.load() | |
| w, h = im.size | |
| if max_pixels and w * h > max_pixels: | |
| s = (max_pixels / (w * h)) ** 0.5 | |
| w2, h2 = max(1, int(w * s)), max(1, int(h * s)) | |
| while w2 * h2 > max_pixels: | |
| if w2 >= h2: | |
| w2 -= 1 | |
| else: | |
| h2 -= 1 | |
| im = im.convert("RGB").resize((w2, h2), Image.LANCZOS) | |
| return im.convert("RGB"), (w, h) | |
| def with_image_placeholders(state: Any, n_images: int) -> Any: | |
| """Add ``<|image_k|>`` references the way the v4 training data writes them, unless the state already has them.""" | |
| if n_images == 0: | |
| return state | |
| text = state if isinstance(state, str) else json.dumps(state, ensure_ascii=False) | |
| if "<|image_1|>" in text: | |
| return state | |
| marks = {"image": "<|image_1|>"} if n_images == 1 else {f"image_{k}": f"<|image_{k}|>" for k in range(1, n_images + 1)} | |
| if isinstance(state, dict): | |
| return {**marks, **state} | |
| if isinstance(state, str): | |
| return " ".join(marks.values()) + ("\n" + state if state else "") | |
| return {**marks, "state": state} | |
| def renormalise(p: Sequence[float]) -> list[float]: | |
| s = float(sum(p)) | |
| return [float(x) / s for x in p] if s > 0 else [1.0 / len(p)] * len(p) | |
| def confidence(p: Sequence[float]) -> float: | |
| k = len(p) | |
| if k <= 1: | |
| return 1.0 | |
| return float(min(1.0, max(0.0, (k * max(p) - 1) / (k - 1)))) | |
| class V4Engine: | |
| def __init__(self, export_dir: str | Path, device: str = "auto", precision: str | None = None, | |
| batch_tokens: int = 65536, max_pixels: int | None = DI_MAX_PIXELS, image_cache: str | Path | None = None, | |
| fast_text: bool = False, cuda_graphs: bool = False): | |
| self.dir = Path(export_dir) | |
| ro_path = self.dir / "macjev_readout.json" | |
| if not ro_path.exists(): | |
| raise FileNotFoundError(f"{ro_path} missing: not a v4 export (scripts/finalize_v4.py writes <out>/hf)") | |
| self.readout_info = ro = json.loads(ro_path.read_text()) | |
| if ro.get("template") != V2_VERSION: | |
| raise ValueError(f"export template {ro.get('template')!r} != this code's {V2_VERSION!r}") | |
| self.device = pick_device(device) | |
| self.precision = precision or ("bf16" if self.device.type == "cuda" else "fp32") | |
| if self.precision not in ("bf16", "fp32"): | |
| raise ValueError("precision must be bf16 or fp32") | |
| dtype = torch.bfloat16 if self.precision == "bf16" else torch.float32 | |
| self.mask_mode = ro["mask_mode"] | |
| impl = None | |
| if self.mask_mode == "block": | |
| from ..train.block_attention import register | |
| impl = register() | |
| t0 = time.perf_counter() | |
| self.vision = bool(ro.get("vision")) | |
| self.bb = bb.load(self.dir, device="cpu", dtype=dtype, attn_implementation=impl, vision=self.vision) | |
| self.bb.model.to(self.device) | |
| self.bb.model.eval() | |
| self.bb.model.config.use_cache = False | |
| self.load_seconds = time.perf_counter() - t0 | |
| self.max_len = int(ro["total_context_limit"]) | |
| self.layout = ro.get("layout") or "sb" | |
| self.renderer = RendererV2(self.bb.tokenizer, "verdict", max_len=self.max_len, layout=self.layout) | |
| self.batch_tokens = int(batch_tokens) | |
| self.max_pixels = int(max_pixels) if max_pixels else None | |
| self.image_loader = self.image_pad_id = None | |
| if self.vision: | |
| from ..vision import ImageLoader | |
| self.image_loader = ImageLoader(self.dir) | |
| self.image_pad_id = self.renderer.vision_ids()[1] | |
| self._tmp = None | |
| if image_cache is None: | |
| self._tmp = tempfile.TemporaryDirectory(prefix="macjev-v4-img-") | |
| image_cache = self._tmp.name | |
| self.image_cache = Path(image_cache) | |
| self.temperatures = {"global": float(ro.get("temperature") or 1.0), **(ro.get("temperatures_by_qtype") or {})} | |
| self.lock = threading.Lock() | |
| files = ro.get("weights_files") or sorted(p.name for p in self.dir.glob("*.safetensors")) | |
| self.model_id = f"{ro.get('run', 'jev-style-v4')}" | |
| self.weights_digest = hashlib.sha256("".join( | |
| (json.loads((self.dir / "SHA256SUMS.json").read_text()).get(f, "") if (self.dir / "SHA256SUMS.json").exists() else f) | |
| for f in files).encode()).hexdigest()[:16] | |
| # fast_text (2026-10-27): text requests with 2+ questions compute their state once (runtime.fast_text); | |
| # single-question requests, requests with images and any request the shared-state check refuses keep the | |
| # reference path (refusals counted in fast_fallbacks) | |
| self.fast = None | |
| self.fast_fallbacks = 0 | |
| if fast_text: | |
| if self.mask_mode != "block" or self.layout != "sb": | |
| raise ValueError("fast_text needs the block mask and the sb layout") | |
| from .fast_text import FastTextScorer | |
| self.fast = FastTextScorer(self.bb, self.renderer, self.device, self.precision, token_budget=self.batch_tokens) | |
| # cuda_graphs (2026-10-27): text rows replay one recorded CUDA graph per padded length (runtime.cuda_graphs): | |
| # single-question requests, and requests whose replays cost less than the fast path (GraphTextScorer.takes); the | |
| # rest keep the paths above. Created after the fast path: its attention also serves the fast path and the reference rows. | |
| self.graphs = None | |
| self.graph_fallbacks = 0 | |
| if cuda_graphs: | |
| if self.mask_mode != "block": | |
| raise ValueError("cuda_graphs needs the block mask") | |
| if self.device.type != "cuda": | |
| raise ValueError("cuda_graphs needs a CUDA device") | |
| from .cuda_graphs import GraphTextScorer | |
| self.graphs = GraphTextScorer(self.bb, self.renderer, self.device, self.precision) | |
| # ---------------------------------------------------------------- info | |
| def provenance(self) -> dict: | |
| ro = self.readout_info | |
| return {"kind": "macjev-v4-export", "path": str(self.dir), "run": ro.get("run"), "selected": ro.get("selected"), | |
| "base": ro.get("base"), "template": ro.get("template"), "mask_mode": self.mask_mode, "layout": self.layout, | |
| "temperatures": self.temperatures, "engine": ENGINE_VERSION, "weights_digest": self.weights_digest} | |
| def runtime(self) -> dict: | |
| return {"device": str(self.device), "precision": self.precision, "torch": torch.__version__, | |
| "total_context_limit": self.max_len, "max_pixels": self.max_pixels, "batch_tokens": self.batch_tokens, | |
| "load_seconds": round(self.load_seconds, 2), "cuda_device": torch.cuda.get_device_name(0) if self.device.type == "cuda" else None, | |
| "fast_text": self.fast is not None, "cuda_graphs": self.graphs.info() if self.graphs is not None else None} | |
| def synchronize(self) -> None: | |
| if self.device.type == "cuda": | |
| torch.cuda.synchronize() | |
| elif self.device.type == "mps": | |
| torch.mps.synchronize() | |
| def temperature(self, qtype: str) -> float: | |
| return float(self.temperatures.get(qtype, self.temperatures["global"])) | |
| def route(self, rows: Sequence[dict]) -> str: | |
| """The scorer for the rows of ONE request: "cuda_graphs", "fast_text" (2+ text questions) or "reference".""" | |
| text = not any(r["pieces"].get("images") for r in rows) | |
| if self.graphs is not None and text and self.graphs.takes(rows, fast_available=self.fast is not None): | |
| return "cuda_graphs" | |
| if self.fast is not None and text and len(rows) >= 2: | |
| return "fast_text" | |
| return "reference" | |
| # ---------------------------------------------------------------- requests -> rows | |
| def _images(self, images: Sequence[Any], root=None) -> tuple[list[dict], list[int]]: | |
| from ..vision import prepare_image | |
| out, pixels = [], [] | |
| for ref in images: | |
| im, _ = _cap(_image_bytes(ref, root), self.max_pixels) | |
| info = prepare_image(im, self.image_cache, budget_px=self.max_pixels or 10 ** 12, fmt="png") | |
| info["path"] = str((self.image_cache / Path(info["path"]).name).resolve()) | |
| out.append(info) | |
| pixels.append(int(info["grid_thw"][1] * info["grid_thw"][2] * 16 * 16)) | |
| return out, pixels | |
| def rows_for(self, parsed: ParsedRequest, images: Sequence[Any] = (), image_root=None, tag: str = "r") -> tuple[list[dict], dict]: | |
| if images and not self.vision: | |
| raise ValueError("this export has no vision tower; the request carries images") | |
| imgs, pixels = self._images(images, image_root) if images else ([], []) | |
| state = with_image_placeholders(parsed.state, len(imgs)) | |
| rows = [] | |
| for q in parsed.questions: | |
| pieces = self.renderer.pieces(state, q.internal, imgs or None) | |
| rendered = self.renderer.assemble(dict(pieces, layout=self.layout)) # raises InputBudgetError | |
| rows.append({"id": f"{tag}:{q.id}", "qtype": q.internal["t"], "names": list(pieces["names"]), "valid_idx": [0], | |
| "soft": None, "pieces": pieces, "n_tokens": {"verdict": len(rendered.ids), "letter": None}, | |
| "render_layout": self.layout, "head_max": self.max_len, "max_len": self.max_len, "weight": 1.0}) | |
| info = {"input_tokens": {q.id: r["n_tokens"]["verdict"] for q, r in zip(parsed.questions, rows)}, | |
| "image_pixels": pixels, "image_tokens": int(sum(int(i["n_tokens"]) for i in imgs)) if imgs else None} | |
| return rows, info | |
| def _answer(self, q: QuestionSpec, logits: Sequence[float], names: list[str]) -> dict: | |
| t = self.temperature(q.internal["t"]) | |
| z = np.asarray(logits, dtype=np.float64) / t | |
| if not np.all(np.isfinite(z)): | |
| raise FloatingPointError("non-finite decision logits") | |
| p = np.exp(z - z.max()) | |
| p = p / p.sum() | |
| by_name = dict(zip(names, p.tolist())) | |
| if q.type == "noul": | |
| return {"type": "noul", "noul": float(by_name["true"])} | |
| if q.type == "choice": | |
| pr = renormalise([by_name[n] for n in q.options]) | |
| i = max(range(len(pr)), key=pr.__getitem__) | |
| return {"type": "choice", "choice": q.options[i], "probabilities": dict(zip(q.options, pr)), "confidence": confidence(pr)} | |
| pr = renormalise([by_name[n] for n in q.options]) | |
| return {"type": "score", "score": float(sum(k * v for k, v in enumerate(pr))), "legend": dict(q.legend), | |
| "probabilities": dict(zip(q.options, pr)), "confidence": confidence(pr)} | |
| # ---------------------------------------------------------------- public | |
| def answer_many(self, requests: Sequence[dict], image_root=None) -> list[Any]: | |
| """``requests``: ``{"state", "questions", "images"?}`` (systemone wire format). Returns, per request, either | |
| ``(response, raw)`` or the exception that request raised (budget / validation), so one bad request never | |
| sinks a batch.""" | |
| prepared, rows, spans = [], [], [] | |
| for n, req in enumerate(requests): | |
| try: | |
| parsed = req if isinstance(req, ParsedRequest) else parse_request(req) | |
| r, info = self.rows_for(parsed, (req.get("images") or ()) if isinstance(req, dict) else (), image_root, tag=f"r{n}") | |
| prepared.append((parsed, info)) | |
| spans.append((len(rows), len(rows) + len(r))) | |
| rows.extend(r) | |
| except Exception as e: # noqa: BLE001 -- returned per request | |
| prepared.append(e) | |
| spans.append(None) | |
| t0 = time.perf_counter() | |
| scorer = {} | |
| with self.lock: | |
| scores: list = [None] * len(rows) | |
| ref = [] | |
| for n, span in enumerate(spans): | |
| if span is None: | |
| continue | |
| # graphs when they take the request; else 2+ text questions share their state (fast path); else the | |
| # reference rows, batched across requests below (one question gains nothing from a shared state) | |
| kind = self.route(rows[span[0]:span[1]]) | |
| if kind == "cuda_graphs": | |
| try: | |
| scores[span[0]:span[1]] = self.graphs.scores(rows[span[0]:span[1]]) | |
| scorer[n] = kind | |
| continue | |
| except ValueError: # a row the recorded graphs refuse: the paths below (graph_fallbacks) | |
| self.graph_fallbacks += 1 | |
| kind = "fast_text" if self.fast is not None and span[1] - span[0] >= 2 else "reference" | |
| if kind == "fast_text": | |
| try: | |
| scores[span[0]:span[1]] = self.fast.scores(rows[span[0]:span[1]]) | |
| scorer[n] = kind | |
| continue | |
| except ValueError: | |
| self.fast_fallbacks += 1 | |
| ref.extend(range(*span)) | |
| scorer[n] = "reference" | |
| if ref: | |
| sub = score_rows(self.bb, self.renderer, [rows[i] for i in ref], self.device, self.precision, self.batch_tokens, | |
| self.mask_mode, image_loader=self.image_loader, image_pad_id=self.image_pad_id) | |
| for i, sc in zip(ref, sub): | |
| scores[i] = sc | |
| self.synchronize() | |
| ms = (time.perf_counter() - t0) * 1000 | |
| out = [] | |
| for n, (item, span) in enumerate(zip(prepared, spans)): | |
| if span is None: | |
| out.append(item) | |
| continue | |
| parsed, info = item | |
| info = {**info, "scorer": scorer.get(n)} | |
| try: | |
| answers = {q.id: self._answer(q, scores[i], rows[i]["names"]) for q, i in zip(parsed.questions, range(*span))} | |
| except Exception as e: # noqa: BLE001 | |
| out.append(e) | |
| continue | |
| n_in = int(sum(info["input_tokens"].values())) | |
| out.append(({"model": self.model_id, "answers": answers, "usage": {"input_tokens": n_in, "output_tokens": 0}}, | |
| {**info, "batch_model_ms": round(ms, 3), "batch_requests": len(requests), | |
| "temperatures": {q.id: self.temperature(q.internal["t"]) for q in parsed.questions}})) | |
| return out | |
| def answer(self, request: dict, image_root=None) -> tuple[dict, dict]: | |
| res = self.answer_many([request], image_root=image_root)[0] | |
| if isinstance(res, Exception): | |
| raise res | |
| return res | |
| def close(self) -> None: | |
| if self._tmp is not None: | |
| self._tmp.cleanup() | |
| self._tmp = None | |