Text Classification
MLX
Safetensors
English
gemma4
mlx-4bit
decision-model
classification
calibration
baseten
apple-silicon
4-bit precision
Instructions to use pratikgx/bison-1-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/bison-1-MLX-4bit with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/bison-1-MLX-4bit --local-dir bison-1-MLX-4bit
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 9,985 Bytes
9eefb05 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 | """MLX decision engine for Baseten bison-1 and groundhog-1.
Full-attention layers run block-bidirectional: the shared prefix is prefilled with no causal
mask, then each question suffix is prefilled against a copy of that cache, also without a causal
mask on the full-attention layers (the suffix attends to the whole prefix and to itself). Sliding
window layers stay causal with the model's window, which is the checkpoint's
`sliding_attention: causal` override and the way Gemma 4 was trained.
The readout is slot_lm: softmax over the LM-head rows of the slot letters at the last token,
computed in float32, then soft-capped like the rest of Gemma 4's logits (final_logit_softcapping=30).
"""
from __future__ import annotations
import copy
import json
import time
from pathlib import Path
from typing import Any, Optional
from .answers import make_choice, make_noul, make_score
from .calibrate import Calibration, softmax
from .prompt import DEFAULT_STYLE, PromptStyle, render_question
from .readout import ReadoutConfig, load_readout_config
from .tokenize import (
load_tokenizer,
prefix_and_suffix_ids,
render_prompt,
template_kwargs_for,
verify_slot_ids,
)
def _text_model(model):
"""Return Gemma4TextModel (has window_size, _make_masks, embed_tokens, hidden states).
mlx-lm wraps it as gemma4.Model.language_model (gemma4_text.Model) .model.
"""
cur = model
seen = set()
while id(cur) not in seen:
seen.add(id(cur))
if hasattr(cur, "window_size") and hasattr(cur, "layers") and hasattr(cur, "_make_masks"):
return cur
if hasattr(cur, "language_model"):
cur = cur.language_model
continue
inner = getattr(cur, "model", None)
if inner is not None and inner is not cur:
cur = inner
continue
break
raise TypeError(f"cannot find the Gemma 4 text model on {type(model).__name__}")
def _logit_softcap(softcap, x):
import mlx.core as mx
return mx.tanh(x / softcap) * softcap
def install_block_bidirectional(model) -> None:
"""Patch Gemma4TextModel._make_masks: full-attention is bidirectional, sliding stays causal+window."""
from mlx_lm.models.base import create_attention_mask
text = _text_model(model)
if getattr(text, "_baseten_block_bidirectional", False):
return
window = text.window_size
def _make_masks(self, h, cache):
mask = {}
masks = []
for layer, c in zip(self.layers, cache):
if layer.layer_type not in mask:
if layer.layer_type == "full_attention":
# Bidirectional within the current segment; cached prefix keys are already
# visible to every suffix query when mask is None.
mask["full_attention"] = None
elif layer.layer_type == "sliding_attention":
mask["sliding_attention"] = create_attention_mask(
h, c, window_size=window
)
else:
mask[layer.layer_type] = create_attention_mask(h, c)
masks.append(mask[layer.layer_type])
return masks
text._make_masks = _make_masks.__get__(text, type(text))
text._baseten_block_bidirectional = True
class DecisionEngine:
def __init__(
self,
model_dir: str | Path,
model=None,
tokenizer=None,
style: PromptStyle = DEFAULT_STYLE,
calibration: Calibration | str | Path | None = None,
):
import mlx.core as mx
from mlx_lm import load
self.model_dir = Path(model_dir)
self.tokenizer = tokenizer or load_tokenizer(self.model_dir)
self.style = style
cfg_path = self.model_dir / "config.json"
model_type = None
if cfg_path.is_file():
raw = json.loads(cfg_path.read_text())
model_type = raw.get("model_type") or (raw.get("text_config") or {}).get("model_type")
self.template_kwargs = template_kwargs_for(model_type or "gemma4")
self.readout: ReadoutConfig = load_readout_config(self.model_dir)
if self.readout.readout != "slot_lm":
raise ValueError(
f"{self.model_dir} uses readout {self.readout.readout!r}; this engine implements slot_lm only"
)
if self.readout.sliding_attention != "causal":
raise ValueError(f"unsupported sliding_attention {self.readout.sliding_attention!r}")
cal_path = self.model_dir / "calibration.json"
if calibration is None:
self.calibration = Calibration.load(cal_path) if cal_path.is_file() else Calibration()
elif isinstance(calibration, (str, Path)):
self.calibration = Calibration.load(calibration)
else:
self.calibration = calibration
self.slot_ids = verify_slot_ids(self.tokenizer)
self.max_slots = len(self.slot_ids)
if model is None:
self.model, _ = load(str(self.model_dir), tokenizer_config={"trust_remote_code": False})
else:
self.model = model
if self.readout.block_bidirectional:
install_block_bidirectional(self.model)
self._init_slot_rows()
mx.eval(self._slot_w)
def _init_slot_rows(self) -> None:
import mlx.core as mx
ids = mx.array(self.slot_ids)
text = _text_model(self.model)
self._embed = text.embed_tokens
self._slot_ids_mx = ids
# Prefer the embedding module's gather: QuantizedEmbedding.dequantizes rows.
# Do not index .weight directly; in 4/8-bit that is packed uint32.
head = getattr(self.model, "lm_head", None) or getattr(text, "lm_head", None)
try:
if head is not None:
if hasattr(head, "scales"):
biases = head.get("biases") if hasattr(head, "get") else getattr(head, "biases", None)
rows = mx.dequantize(
head.weight[ids],
scales=head.scales[ids],
biases=biases[ids] if biases is not None else None,
group_size=head.group_size,
bits=head.bits,
mode=head.mode,
)
else:
rows = head.weight[ids]
else:
rows = text.embed_tokens(ids)
self._slot_w = rows.astype(mx.float32)
except Exception:
self._slot_w = None
def _slot_logits(self, hidden_last):
"""hidden_last: [H] or [B, H] final-norm output -> float32 slot logits, soft-capped."""
import mlx.core as mx
h = hidden_last.astype(mx.float32)
if h.ndim == 1:
h = h[None, :]
if self._slot_w is not None:
logits = h @ self._slot_w.T
else:
# Quantized or wrapped head: use the official last-token linear path, then gather slot rows.
h = hidden_last if hidden_last.ndim > 1 else hidden_last[None]
if hasattr(self.model, "lm_head"):
full = self.model.lm_head(h)
else:
full = self._embed.as_linear(h)
full = full.astype(mx.float32)
logits = full[:, self._slot_ids_mx]
if self.readout.logit_softcap:
logits = _logit_softcap(self.readout.logit_softcap, logits)
return logits
def _forward_hidden(self, token_ids: list[int], cache):
import mlx.core as mx
text = _text_model(self.model)
x = mx.array([token_ids])
hidden = text(x, cache=cache)
mx.eval(hidden)
return hidden
def decide(self, state: Any, questions: dict[str, Any], debug: bool = False) -> dict[str, Any]:
import mlx.core as mx
t0 = time.perf_counter()
rendered = [render_question(qid, q, style=self.style) for qid, q in questions.items()]
prefix_ids, suffixes, prompts = prefix_and_suffix_ids(
self.tokenizer, state, rendered, self.style, self.template_kwargs
)
t1 = time.perf_counter()
base = self.model.make_cache()
self._forward_hidden(prefix_ids, base)
mx.eval([c.state for c in base if getattr(c, "state", None) is not None])
answers: dict[str, Any] = {}
diag: dict[str, Any] = {}
for r, suf in zip(rendered, suffixes):
cache = copy.deepcopy(base)
hidden = self._forward_hidden(suf, cache)
logits = self._slot_logits(hidden[0, -1])
mx.eval(logits)
slot = logits[0, : r.n_slots].tolist()
probs = softmax(slot, self.calibration.temperature(r.kind, r.n_slots))
if r.kind == "noul":
answers[r.qid] = make_noul(probs[0])
elif r.kind == "choice":
answers[r.qid] = make_choice(r.labels, probs)
else:
answers[r.qid] = make_score(r.descriptions, probs)
if debug:
diag[r.qid] = {"slot_logits": slot, "temperature": self.calibration.temperature(r.kind, r.n_slots)}
del cache, hidden
t2 = time.perf_counter()
total = len(prefix_ids) + sum(len(s) for s in suffixes)
out = {
"answers": answers,
"usage": {"input_tokens": total, "output_tokens": 0},
"qev": {
"prompt_version": self.style.name,
"prefix_tokens": len(prefix_ids),
"branches": len(rendered),
"attention": "blocks" if self.readout.block_bidirectional else "causal",
"readout": self.readout.readout,
"timing": {"render_s": t1 - t0, "forward_s": t2 - t1, "total_s": time.perf_counter() - t0},
},
}
if debug:
out["qev"]["diagnostics"] = diag
out["qev"]["prompt"] = prompts[0]
return out
|