"""MLX decision engine for Baseten bison-1 and groundhog-1. Full-attention layers run block-bidirectional: the shared prefix is prefilled with no causal mask, then each question suffix is prefilled against a copy of that cache, also without a causal mask on the full-attention layers (the suffix attends to the whole prefix and to itself). Sliding window layers stay causal with the model's window, which is the checkpoint's `sliding_attention: causal` override and the way Gemma 4 was trained. The readout is slot_lm: softmax over the LM-head rows of the slot letters at the last token, computed in float32, then soft-capped like the rest of Gemma 4's logits (final_logit_softcapping=30). """ from __future__ import annotations import copy import json import time from pathlib import Path from typing import Any, Optional from .answers import make_choice, make_noul, make_score from .calibrate import Calibration, softmax from .prompt import DEFAULT_STYLE, PromptStyle, render_question from .readout import ReadoutConfig, load_readout_config from .tokenize import ( load_tokenizer, prefix_and_suffix_ids, render_prompt, template_kwargs_for, verify_slot_ids, ) def _text_model(model): """Return Gemma4TextModel (has window_size, _make_masks, embed_tokens, hidden states). mlx-lm wraps it as gemma4.Model.language_model (gemma4_text.Model) .model. """ cur = model seen = set() while id(cur) not in seen: seen.add(id(cur)) if hasattr(cur, "window_size") and hasattr(cur, "layers") and hasattr(cur, "_make_masks"): return cur if hasattr(cur, "language_model"): cur = cur.language_model continue inner = getattr(cur, "model", None) if inner is not None and inner is not cur: cur = inner continue break raise TypeError(f"cannot find the Gemma 4 text model on {type(model).__name__}") def _logit_softcap(softcap, x): import mlx.core as mx return mx.tanh(x / softcap) * softcap def install_block_bidirectional(model) -> None: """Patch Gemma4TextModel._make_masks: full-attention is bidirectional, sliding stays causal+window.""" from mlx_lm.models.base import create_attention_mask text = _text_model(model) if getattr(text, "_baseten_block_bidirectional", False): return window = text.window_size def _make_masks(self, h, cache): mask = {} masks = [] for layer, c in zip(self.layers, cache): if layer.layer_type not in mask: if layer.layer_type == "full_attention": # Bidirectional within the current segment; cached prefix keys are already # visible to every suffix query when mask is None. mask["full_attention"] = None elif layer.layer_type == "sliding_attention": mask["sliding_attention"] = create_attention_mask( h, c, window_size=window ) else: mask[layer.layer_type] = create_attention_mask(h, c) masks.append(mask[layer.layer_type]) return masks text._make_masks = _make_masks.__get__(text, type(text)) text._baseten_block_bidirectional = True class DecisionEngine: def __init__( self, model_dir: str | Path, model=None, tokenizer=None, style: PromptStyle = DEFAULT_STYLE, calibration: Calibration | str | Path | None = None, ): import mlx.core as mx from mlx_lm import load self.model_dir = Path(model_dir) self.tokenizer = tokenizer or load_tokenizer(self.model_dir) self.style = style cfg_path = self.model_dir / "config.json" model_type = None if cfg_path.is_file(): raw = json.loads(cfg_path.read_text()) model_type = raw.get("model_type") or (raw.get("text_config") or {}).get("model_type") self.template_kwargs = template_kwargs_for(model_type or "gemma4") self.readout: ReadoutConfig = load_readout_config(self.model_dir) if self.readout.readout != "slot_lm": raise ValueError( f"{self.model_dir} uses readout {self.readout.readout!r}; this engine implements slot_lm only" ) if self.readout.sliding_attention != "causal": raise ValueError(f"unsupported sliding_attention {self.readout.sliding_attention!r}") cal_path = self.model_dir / "calibration.json" if calibration is None: self.calibration = Calibration.load(cal_path) if cal_path.is_file() else Calibration() elif isinstance(calibration, (str, Path)): self.calibration = Calibration.load(calibration) else: self.calibration = calibration self.slot_ids = verify_slot_ids(self.tokenizer) self.max_slots = len(self.slot_ids) if model is None: self.model, _ = load(str(self.model_dir), tokenizer_config={"trust_remote_code": False}) else: self.model = model if self.readout.block_bidirectional: install_block_bidirectional(self.model) self._init_slot_rows() mx.eval(self._slot_w) def _init_slot_rows(self) -> None: import mlx.core as mx ids = mx.array(self.slot_ids) text = _text_model(self.model) self._embed = text.embed_tokens self._slot_ids_mx = ids # Prefer the embedding module's gather: QuantizedEmbedding.dequantizes rows. # Do not index .weight directly; in 4/8-bit that is packed uint32. head = getattr(self.model, "lm_head", None) or getattr(text, "lm_head", None) try: if head is not None: if hasattr(head, "scales"): biases = head.get("biases") if hasattr(head, "get") else getattr(head, "biases", None) rows = mx.dequantize( head.weight[ids], scales=head.scales[ids], biases=biases[ids] if biases is not None else None, group_size=head.group_size, bits=head.bits, mode=head.mode, ) else: rows = head.weight[ids] else: rows = text.embed_tokens(ids) self._slot_w = rows.astype(mx.float32) except Exception: self._slot_w = None def _slot_logits(self, hidden_last): """hidden_last: [H] or [B, H] final-norm output -> float32 slot logits, soft-capped.""" import mlx.core as mx h = hidden_last.astype(mx.float32) if h.ndim == 1: h = h[None, :] if self._slot_w is not None: logits = h @ self._slot_w.T else: # Quantized or wrapped head: use the official last-token linear path, then gather slot rows. h = hidden_last if hidden_last.ndim > 1 else hidden_last[None] if hasattr(self.model, "lm_head"): full = self.model.lm_head(h) else: full = self._embed.as_linear(h) full = full.astype(mx.float32) logits = full[:, self._slot_ids_mx] if self.readout.logit_softcap: logits = _logit_softcap(self.readout.logit_softcap, logits) return logits def _forward_hidden(self, token_ids: list[int], cache): import mlx.core as mx text = _text_model(self.model) x = mx.array([token_ids]) hidden = text(x, cache=cache) mx.eval(hidden) return hidden def decide(self, state: Any, questions: dict[str, Any], debug: bool = False) -> dict[str, Any]: import mlx.core as mx t0 = time.perf_counter() rendered = [render_question(qid, q, style=self.style) for qid, q in questions.items()] prefix_ids, suffixes, prompts = prefix_and_suffix_ids( self.tokenizer, state, rendered, self.style, self.template_kwargs ) t1 = time.perf_counter() base = self.model.make_cache() self._forward_hidden(prefix_ids, base) mx.eval([c.state for c in base if getattr(c, "state", None) is not None]) answers: dict[str, Any] = {} diag: dict[str, Any] = {} for r, suf in zip(rendered, suffixes): cache = copy.deepcopy(base) hidden = self._forward_hidden(suf, cache) logits = self._slot_logits(hidden[0, -1]) mx.eval(logits) slot = logits[0, : r.n_slots].tolist() probs = softmax(slot, self.calibration.temperature(r.kind, r.n_slots)) if r.kind == "noul": answers[r.qid] = make_noul(probs[0]) elif r.kind == "choice": answers[r.qid] = make_choice(r.labels, probs) else: answers[r.qid] = make_score(r.descriptions, probs) if debug: diag[r.qid] = {"slot_logits": slot, "temperature": self.calibration.temperature(r.kind, r.n_slots)} del cache, hidden t2 = time.perf_counter() total = len(prefix_ids) + sum(len(s) for s in suffixes) out = { "answers": answers, "usage": {"input_tokens": total, "output_tokens": 0}, "qev": { "prompt_version": self.style.name, "prefix_tokens": len(prefix_ids), "branches": len(rendered), "attention": "blocks" if self.readout.block_bidirectional else "causal", "readout": self.readout.readout, "timing": {"render_s": t1 - t0, "forward_s": t2 - t1, "total_s": time.perf_counter() - t0}, }, } if debug: out["qev"]["diagnostics"] = diag out["qev"]["prompt"] = prompts[0] return out