bison-1-MLX-4bit / engine /engine_mlx.py
pratikgx's picture
Add bison-1 MLX 4-bit weights, System One engine, and model card
9eefb05 verified
Raw History Blame Contribute Delete
9.99 kB
"""MLX decision engine for Baseten bison-1 and groundhog-1.
Full-attention layers run block-bidirectional: the shared prefix is prefilled with no causal
mask, then each question suffix is prefilled against a copy of that cache, also without a causal
mask on the full-attention layers (the suffix attends to the whole prefix and to itself). Sliding
window layers stay causal with the model's window, which is the checkpoint's
`sliding_attention: causal` override and the way Gemma 4 was trained.
The readout is slot_lm: softmax over the LM-head rows of the slot letters at the last token,
computed in float32, then soft-capped like the rest of Gemma 4's logits (final_logit_softcapping=30).
"""
from __future__ import annotations
import copy
import json
import time
from pathlib import Path
from typing import Any, Optional
from .answers import make_choice, make_noul, make_score
from .calibrate import Calibration, softmax
from .prompt import DEFAULT_STYLE, PromptStyle, render_question
from .readout import ReadoutConfig, load_readout_config
from .tokenize import (
load_tokenizer,
prefix_and_suffix_ids,
render_prompt,
template_kwargs_for,
verify_slot_ids,
)
def _text_model(model):
"""Return Gemma4TextModel (has window_size, _make_masks, embed_tokens, hidden states).
mlx-lm wraps it as gemma4.Model.language_model (gemma4_text.Model) .model.
"""
cur = model
seen = set()
while id(cur) not in seen:
seen.add(id(cur))
if hasattr(cur, "window_size") and hasattr(cur, "layers") and hasattr(cur, "_make_masks"):
return cur
if hasattr(cur, "language_model"):
cur = cur.language_model
continue
inner = getattr(cur, "model", None)
if inner is not None and inner is not cur:
cur = inner
continue
break
raise TypeError(f"cannot find the Gemma 4 text model on {type(model).__name__}")
def _logit_softcap(softcap, x):
import mlx.core as mx
return mx.tanh(x / softcap) * softcap
def install_block_bidirectional(model) -> None:
"""Patch Gemma4TextModel._make_masks: full-attention is bidirectional, sliding stays causal+window."""
from mlx_lm.models.base import create_attention_mask
text = _text_model(model)
if getattr(text, "_baseten_block_bidirectional", False):
return
window = text.window_size
def _make_masks(self, h, cache):
mask = {}
masks = []
for layer, c in zip(self.layers, cache):
if layer.layer_type not in mask:
if layer.layer_type == "full_attention":
# Bidirectional within the current segment; cached prefix keys are already
# visible to every suffix query when mask is None.
mask["full_attention"] = None
elif layer.layer_type == "sliding_attention":
mask["sliding_attention"] = create_attention_mask(
h, c, window_size=window
)
else:
mask[layer.layer_type] = create_attention_mask(h, c)
masks.append(mask[layer.layer_type])
return masks
text._make_masks = _make_masks.__get__(text, type(text))
text._baseten_block_bidirectional = True
class DecisionEngine:
def __init__(
self,
model_dir: str | Path,
model=None,
tokenizer=None,
style: PromptStyle = DEFAULT_STYLE,
calibration: Calibration | str | Path | None = None,
):
import mlx.core as mx
from mlx_lm import load
self.model_dir = Path(model_dir)
self.tokenizer = tokenizer or load_tokenizer(self.model_dir)
self.style = style
cfg_path = self.model_dir / "config.json"
model_type = None
if cfg_path.is_file():
raw = json.loads(cfg_path.read_text())
model_type = raw.get("model_type") or (raw.get("text_config") or {}).get("model_type")
self.template_kwargs = template_kwargs_for(model_type or "gemma4")
self.readout: ReadoutConfig = load_readout_config(self.model_dir)
if self.readout.readout != "slot_lm":
raise ValueError(
f"{self.model_dir} uses readout {self.readout.readout!r}; this engine implements slot_lm only"
)
if self.readout.sliding_attention != "causal":
raise ValueError(f"unsupported sliding_attention {self.readout.sliding_attention!r}")
cal_path = self.model_dir / "calibration.json"
if calibration is None:
self.calibration = Calibration.load(cal_path) if cal_path.is_file() else Calibration()
elif isinstance(calibration, (str, Path)):
self.calibration = Calibration.load(calibration)
else:
self.calibration = calibration
self.slot_ids = verify_slot_ids(self.tokenizer)
self.max_slots = len(self.slot_ids)
if model is None:
self.model, _ = load(str(self.model_dir), tokenizer_config={"trust_remote_code": False})
else:
self.model = model
if self.readout.block_bidirectional:
install_block_bidirectional(self.model)
self._init_slot_rows()
mx.eval(self._slot_w)
def _init_slot_rows(self) -> None:
import mlx.core as mx
ids = mx.array(self.slot_ids)
text = _text_model(self.model)
self._embed = text.embed_tokens
self._slot_ids_mx = ids
# Prefer the embedding module's gather: QuantizedEmbedding.dequantizes rows.
# Do not index .weight directly; in 4/8-bit that is packed uint32.
head = getattr(self.model, "lm_head", None) or getattr(text, "lm_head", None)
try:
if head is not None:
if hasattr(head, "scales"):
biases = head.get("biases") if hasattr(head, "get") else getattr(head, "biases", None)
rows = mx.dequantize(
head.weight[ids],
scales=head.scales[ids],
biases=biases[ids] if biases is not None else None,
group_size=head.group_size,
bits=head.bits,
mode=head.mode,
)
else:
rows = head.weight[ids]
else:
rows = text.embed_tokens(ids)
self._slot_w = rows.astype(mx.float32)
except Exception:
self._slot_w = None
def _slot_logits(self, hidden_last):
"""hidden_last: [H] or [B, H] final-norm output -> float32 slot logits, soft-capped."""
import mlx.core as mx
h = hidden_last.astype(mx.float32)
if h.ndim == 1:
h = h[None, :]
if self._slot_w is not None:
logits = h @ self._slot_w.T
else:
# Quantized or wrapped head: use the official last-token linear path, then gather slot rows.
h = hidden_last if hidden_last.ndim > 1 else hidden_last[None]
if hasattr(self.model, "lm_head"):
full = self.model.lm_head(h)
else:
full = self._embed.as_linear(h)
full = full.astype(mx.float32)
logits = full[:, self._slot_ids_mx]
if self.readout.logit_softcap:
logits = _logit_softcap(self.readout.logit_softcap, logits)
return logits
def _forward_hidden(self, token_ids: list[int], cache):
import mlx.core as mx
text = _text_model(self.model)
x = mx.array([token_ids])
hidden = text(x, cache=cache)
mx.eval(hidden)
return hidden
def decide(self, state: Any, questions: dict[str, Any], debug: bool = False) -> dict[str, Any]:
import mlx.core as mx
t0 = time.perf_counter()
rendered = [render_question(qid, q, style=self.style) for qid, q in questions.items()]
prefix_ids, suffixes, prompts = prefix_and_suffix_ids(
self.tokenizer, state, rendered, self.style, self.template_kwargs
)
t1 = time.perf_counter()
base = self.model.make_cache()
self._forward_hidden(prefix_ids, base)
mx.eval([c.state for c in base if getattr(c, "state", None) is not None])
answers: dict[str, Any] = {}
diag: dict[str, Any] = {}
for r, suf in zip(rendered, suffixes):
cache = copy.deepcopy(base)
hidden = self._forward_hidden(suf, cache)
logits = self._slot_logits(hidden[0, -1])
mx.eval(logits)
slot = logits[0, : r.n_slots].tolist()
probs = softmax(slot, self.calibration.temperature(r.kind, r.n_slots))
if r.kind == "noul":
answers[r.qid] = make_noul(probs[0])
elif r.kind == "choice":
answers[r.qid] = make_choice(r.labels, probs)
else:
answers[r.qid] = make_score(r.descriptions, probs)
if debug:
diag[r.qid] = {"slot_logits": slot, "temperature": self.calibration.temperature(r.kind, r.n_slots)}
del cache, hidden
t2 = time.perf_counter()
total = len(prefix_ids) + sum(len(s) for s in suffixes)
out = {
"answers": answers,
"usage": {"input_tokens": total, "output_tokens": 0},
"qev": {
"prompt_version": self.style.name,
"prefix_tokens": len(prefix_ids),
"branches": len(rendered),
"attention": "blocks" if self.readout.block_bidirectional else "causal",
"readout": self.readout.readout,
"timing": {"render_s": t1 - t0, "forward_s": t2 - t1, "total_s": time.perf_counter() - t0},
},
}
if debug:
out["qev"]["diagnostics"] = diag
out["qev"]["prompt"] = prompts[0]
return out