File size: 9,985 Bytes
9eefb05
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
"""MLX decision engine for Baseten bison-1 and groundhog-1.

Full-attention layers run block-bidirectional: the shared prefix is prefilled with no causal
mask, then each question suffix is prefilled against a copy of that cache, also without a causal
mask on the full-attention layers (the suffix attends to the whole prefix and to itself). Sliding
window layers stay causal with the model's window, which is the checkpoint's
`sliding_attention: causal` override and the way Gemma 4 was trained.

The readout is slot_lm: softmax over the LM-head rows of the slot letters at the last token,
computed in float32, then soft-capped like the rest of Gemma 4's logits (final_logit_softcapping=30).
"""
from __future__ import annotations

import copy
import json
import time
from pathlib import Path
from typing import Any, Optional

from .answers import make_choice, make_noul, make_score
from .calibrate import Calibration, softmax
from .prompt import DEFAULT_STYLE, PromptStyle, render_question
from .readout import ReadoutConfig, load_readout_config
from .tokenize import (
    load_tokenizer,
    prefix_and_suffix_ids,
    render_prompt,
    template_kwargs_for,
    verify_slot_ids,
)


def _text_model(model):
    """Return Gemma4TextModel (has window_size, _make_masks, embed_tokens, hidden states).

    mlx-lm wraps it as gemma4.Model.language_model (gemma4_text.Model) .model.
    """
    cur = model
    seen = set()
    while id(cur) not in seen:
        seen.add(id(cur))
        if hasattr(cur, "window_size") and hasattr(cur, "layers") and hasattr(cur, "_make_masks"):
            return cur
        if hasattr(cur, "language_model"):
            cur = cur.language_model
            continue
        inner = getattr(cur, "model", None)
        if inner is not None and inner is not cur:
            cur = inner
            continue
        break
    raise TypeError(f"cannot find the Gemma 4 text model on {type(model).__name__}")


def _logit_softcap(softcap, x):
    import mlx.core as mx

    return mx.tanh(x / softcap) * softcap


def install_block_bidirectional(model) -> None:
    """Patch Gemma4TextModel._make_masks: full-attention is bidirectional, sliding stays causal+window."""
    from mlx_lm.models.base import create_attention_mask

    text = _text_model(model)
    if getattr(text, "_baseten_block_bidirectional", False):
        return
    window = text.window_size

    def _make_masks(self, h, cache):
        mask = {}
        masks = []
        for layer, c in zip(self.layers, cache):
            if layer.layer_type not in mask:
                if layer.layer_type == "full_attention":
                    # Bidirectional within the current segment; cached prefix keys are already
                    # visible to every suffix query when mask is None.
                    mask["full_attention"] = None
                elif layer.layer_type == "sliding_attention":
                    mask["sliding_attention"] = create_attention_mask(
                        h, c, window_size=window
                    )
                else:
                    mask[layer.layer_type] = create_attention_mask(h, c)
            masks.append(mask[layer.layer_type])
        return masks

    text._make_masks = _make_masks.__get__(text, type(text))
    text._baseten_block_bidirectional = True


class DecisionEngine:
    def __init__(
        self,
        model_dir: str | Path,
        model=None,
        tokenizer=None,
        style: PromptStyle = DEFAULT_STYLE,
        calibration: Calibration | str | Path | None = None,
    ):
        import mlx.core as mx
        from mlx_lm import load

        self.model_dir = Path(model_dir)
        self.tokenizer = tokenizer or load_tokenizer(self.model_dir)
        self.style = style
        cfg_path = self.model_dir / "config.json"
        model_type = None
        if cfg_path.is_file():
            raw = json.loads(cfg_path.read_text())
            model_type = raw.get("model_type") or (raw.get("text_config") or {}).get("model_type")
        self.template_kwargs = template_kwargs_for(model_type or "gemma4")
        self.readout: ReadoutConfig = load_readout_config(self.model_dir)
        if self.readout.readout != "slot_lm":
            raise ValueError(
                f"{self.model_dir} uses readout {self.readout.readout!r}; this engine implements slot_lm only"
            )
        if self.readout.sliding_attention != "causal":
            raise ValueError(f"unsupported sliding_attention {self.readout.sliding_attention!r}")
        cal_path = self.model_dir / "calibration.json"
        if calibration is None:
            self.calibration = Calibration.load(cal_path) if cal_path.is_file() else Calibration()
        elif isinstance(calibration, (str, Path)):
            self.calibration = Calibration.load(calibration)
        else:
            self.calibration = calibration
        self.slot_ids = verify_slot_ids(self.tokenizer)
        self.max_slots = len(self.slot_ids)
        if model is None:
            self.model, _ = load(str(self.model_dir), tokenizer_config={"trust_remote_code": False})
        else:
            self.model = model
        if self.readout.block_bidirectional:
            install_block_bidirectional(self.model)
        self._init_slot_rows()
        mx.eval(self._slot_w)

    def _init_slot_rows(self) -> None:
        import mlx.core as mx

        ids = mx.array(self.slot_ids)
        text = _text_model(self.model)
        self._embed = text.embed_tokens
        self._slot_ids_mx = ids
        # Prefer the embedding module's gather: QuantizedEmbedding.dequantizes rows.
        # Do not index .weight directly; in 4/8-bit that is packed uint32.
        head = getattr(self.model, "lm_head", None) or getattr(text, "lm_head", None)
        try:
            if head is not None:
                if hasattr(head, "scales"):
                    biases = head.get("biases") if hasattr(head, "get") else getattr(head, "biases", None)
                    rows = mx.dequantize(
                        head.weight[ids],
                        scales=head.scales[ids],
                        biases=biases[ids] if biases is not None else None,
                        group_size=head.group_size,
                        bits=head.bits,
                        mode=head.mode,
                    )
                else:
                    rows = head.weight[ids]
            else:
                rows = text.embed_tokens(ids)
            self._slot_w = rows.astype(mx.float32)
        except Exception:
            self._slot_w = None

    def _slot_logits(self, hidden_last):
        """hidden_last: [H] or [B, H] final-norm output -> float32 slot logits, soft-capped."""
        import mlx.core as mx

        h = hidden_last.astype(mx.float32)
        if h.ndim == 1:
            h = h[None, :]
        if self._slot_w is not None:
            logits = h @ self._slot_w.T
        else:
            # Quantized or wrapped head: use the official last-token linear path, then gather slot rows.
            h = hidden_last if hidden_last.ndim > 1 else hidden_last[None]
            if hasattr(self.model, "lm_head"):
                full = self.model.lm_head(h)
            else:
                full = self._embed.as_linear(h)
            full = full.astype(mx.float32)
            logits = full[:, self._slot_ids_mx]
        if self.readout.logit_softcap:
            logits = _logit_softcap(self.readout.logit_softcap, logits)
        return logits

    def _forward_hidden(self, token_ids: list[int], cache):
        import mlx.core as mx

        text = _text_model(self.model)
        x = mx.array([token_ids])
        hidden = text(x, cache=cache)
        mx.eval(hidden)
        return hidden

    def decide(self, state: Any, questions: dict[str, Any], debug: bool = False) -> dict[str, Any]:
        import mlx.core as mx

        t0 = time.perf_counter()
        rendered = [render_question(qid, q, style=self.style) for qid, q in questions.items()]
        prefix_ids, suffixes, prompts = prefix_and_suffix_ids(
            self.tokenizer, state, rendered, self.style, self.template_kwargs
        )
        t1 = time.perf_counter()
        base = self.model.make_cache()
        self._forward_hidden(prefix_ids, base)
        mx.eval([c.state for c in base if getattr(c, "state", None) is not None])
        answers: dict[str, Any] = {}
        diag: dict[str, Any] = {}
        for r, suf in zip(rendered, suffixes):
            cache = copy.deepcopy(base)
            hidden = self._forward_hidden(suf, cache)
            logits = self._slot_logits(hidden[0, -1])
            mx.eval(logits)
            slot = logits[0, : r.n_slots].tolist()
            probs = softmax(slot, self.calibration.temperature(r.kind, r.n_slots))
            if r.kind == "noul":
                answers[r.qid] = make_noul(probs[0])
            elif r.kind == "choice":
                answers[r.qid] = make_choice(r.labels, probs)
            else:
                answers[r.qid] = make_score(r.descriptions, probs)
            if debug:
                diag[r.qid] = {"slot_logits": slot, "temperature": self.calibration.temperature(r.kind, r.n_slots)}
            del cache, hidden
        t2 = time.perf_counter()
        total = len(prefix_ids) + sum(len(s) for s in suffixes)
        out = {
            "answers": answers,
            "usage": {"input_tokens": total, "output_tokens": 0},
            "qev": {
                "prompt_version": self.style.name,
                "prefix_tokens": len(prefix_ids),
                "branches": len(rendered),
                "attention": "blocks" if self.readout.block_bidirectional else "causal",
                "readout": self.readout.readout,
                "timing": {"render_s": t1 - t0, "forward_s": t2 - t1, "total_s": time.perf_counter() - t0},
            },
        }
        if debug:
            out["qev"]["diagnostics"] = diag
            out["qev"]["prompt"] = prompts[0]
        return out