Text Classification
MLX
Safetensors
English
gemma4
mlx-4bit
decision-model
classification
calibration
baseten
apple-silicon
4-bit precision
Instructions to use pratikgx/bison-1-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/bison-1-MLX-4bit with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/bison-1-MLX-4bit --local-dir bison-1-MLX-4bit
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download engine/engine_mlx.py from pratikgx/bison-1-MLX-4bit: direct link, hf CLI and curl.
- Browser
- Download file 9.99 kB
-
https://huggingface.co/pratikgx/bison-1-MLX-4bit/resolve/main/engine/engine_mlx.py
- Command line
-
hf download hf://pratikgx/bison-1-MLX-4bit/engine/engine_mlx.py
-
curl -L -o engine_mlx.py https://huggingface.co/pratikgx/bison-1-MLX-4bit/resolve/main/engine/engine_mlx.py
9.99 kB
| """MLX decision engine for Baseten bison-1 and groundhog-1. | |
| Full-attention layers run block-bidirectional: the shared prefix is prefilled with no causal | |
| mask, then each question suffix is prefilled against a copy of that cache, also without a causal | |
| mask on the full-attention layers (the suffix attends to the whole prefix and to itself). Sliding | |
| window layers stay causal with the model's window, which is the checkpoint's | |
| `sliding_attention: causal` override and the way Gemma 4 was trained. | |
| The readout is slot_lm: softmax over the LM-head rows of the slot letters at the last token, | |
| computed in float32, then soft-capped like the rest of Gemma 4's logits (final_logit_softcapping=30). | |
| """ | |
| from __future__ import annotations | |
| import copy | |
| import json | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Optional | |
| from .answers import make_choice, make_noul, make_score | |
| from .calibrate import Calibration, softmax | |
| from .prompt import DEFAULT_STYLE, PromptStyle, render_question | |
| from .readout import ReadoutConfig, load_readout_config | |
| from .tokenize import ( | |
| load_tokenizer, | |
| prefix_and_suffix_ids, | |
| render_prompt, | |
| template_kwargs_for, | |
| verify_slot_ids, | |
| ) | |
| def _text_model(model): | |
| """Return Gemma4TextModel (has window_size, _make_masks, embed_tokens, hidden states). | |
| mlx-lm wraps it as gemma4.Model.language_model (gemma4_text.Model) .model. | |
| """ | |
| cur = model | |
| seen = set() | |
| while id(cur) not in seen: | |
| seen.add(id(cur)) | |
| if hasattr(cur, "window_size") and hasattr(cur, "layers") and hasattr(cur, "_make_masks"): | |
| return cur | |
| if hasattr(cur, "language_model"): | |
| cur = cur.language_model | |
| continue | |
| inner = getattr(cur, "model", None) | |
| if inner is not None and inner is not cur: | |
| cur = inner | |
| continue | |
| break | |
| raise TypeError(f"cannot find the Gemma 4 text model on {type(model).__name__}") | |
| def _logit_softcap(softcap, x): | |
| import mlx.core as mx | |
| return mx.tanh(x / softcap) * softcap | |
| def install_block_bidirectional(model) -> None: | |
| """Patch Gemma4TextModel._make_masks: full-attention is bidirectional, sliding stays causal+window.""" | |
| from mlx_lm.models.base import create_attention_mask | |
| text = _text_model(model) | |
| if getattr(text, "_baseten_block_bidirectional", False): | |
| return | |
| window = text.window_size | |
| def _make_masks(self, h, cache): | |
| mask = {} | |
| masks = [] | |
| for layer, c in zip(self.layers, cache): | |
| if layer.layer_type not in mask: | |
| if layer.layer_type == "full_attention": | |
| # Bidirectional within the current segment; cached prefix keys are already | |
| # visible to every suffix query when mask is None. | |
| mask["full_attention"] = None | |
| elif layer.layer_type == "sliding_attention": | |
| mask["sliding_attention"] = create_attention_mask( | |
| h, c, window_size=window | |
| ) | |
| else: | |
| mask[layer.layer_type] = create_attention_mask(h, c) | |
| masks.append(mask[layer.layer_type]) | |
| return masks | |
| text._make_masks = _make_masks.__get__(text, type(text)) | |
| text._baseten_block_bidirectional = True | |
| class DecisionEngine: | |
| def __init__( | |
| self, | |
| model_dir: str | Path, | |
| model=None, | |
| tokenizer=None, | |
| style: PromptStyle = DEFAULT_STYLE, | |
| calibration: Calibration | str | Path | None = None, | |
| ): | |
| import mlx.core as mx | |
| from mlx_lm import load | |
| self.model_dir = Path(model_dir) | |
| self.tokenizer = tokenizer or load_tokenizer(self.model_dir) | |
| self.style = style | |
| cfg_path = self.model_dir / "config.json" | |
| model_type = None | |
| if cfg_path.is_file(): | |
| raw = json.loads(cfg_path.read_text()) | |
| model_type = raw.get("model_type") or (raw.get("text_config") or {}).get("model_type") | |
| self.template_kwargs = template_kwargs_for(model_type or "gemma4") | |
| self.readout: ReadoutConfig = load_readout_config(self.model_dir) | |
| if self.readout.readout != "slot_lm": | |
| raise ValueError( | |
| f"{self.model_dir} uses readout {self.readout.readout!r}; this engine implements slot_lm only" | |
| ) | |
| if self.readout.sliding_attention != "causal": | |
| raise ValueError(f"unsupported sliding_attention {self.readout.sliding_attention!r}") | |
| cal_path = self.model_dir / "calibration.json" | |
| if calibration is None: | |
| self.calibration = Calibration.load(cal_path) if cal_path.is_file() else Calibration() | |
| elif isinstance(calibration, (str, Path)): | |
| self.calibration = Calibration.load(calibration) | |
| else: | |
| self.calibration = calibration | |
| self.slot_ids = verify_slot_ids(self.tokenizer) | |
| self.max_slots = len(self.slot_ids) | |
| if model is None: | |
| self.model, _ = load(str(self.model_dir), tokenizer_config={"trust_remote_code": False}) | |
| else: | |
| self.model = model | |
| if self.readout.block_bidirectional: | |
| install_block_bidirectional(self.model) | |
| self._init_slot_rows() | |
| mx.eval(self._slot_w) | |
| def _init_slot_rows(self) -> None: | |
| import mlx.core as mx | |
| ids = mx.array(self.slot_ids) | |
| text = _text_model(self.model) | |
| self._embed = text.embed_tokens | |
| self._slot_ids_mx = ids | |
| # Prefer the embedding module's gather: QuantizedEmbedding.dequantizes rows. | |
| # Do not index .weight directly; in 4/8-bit that is packed uint32. | |
| head = getattr(self.model, "lm_head", None) or getattr(text, "lm_head", None) | |
| try: | |
| if head is not None: | |
| if hasattr(head, "scales"): | |
| biases = head.get("biases") if hasattr(head, "get") else getattr(head, "biases", None) | |
| rows = mx.dequantize( | |
| head.weight[ids], | |
| scales=head.scales[ids], | |
| biases=biases[ids] if biases is not None else None, | |
| group_size=head.group_size, | |
| bits=head.bits, | |
| mode=head.mode, | |
| ) | |
| else: | |
| rows = head.weight[ids] | |
| else: | |
| rows = text.embed_tokens(ids) | |
| self._slot_w = rows.astype(mx.float32) | |
| except Exception: | |
| self._slot_w = None | |
| def _slot_logits(self, hidden_last): | |
| """hidden_last: [H] or [B, H] final-norm output -> float32 slot logits, soft-capped.""" | |
| import mlx.core as mx | |
| h = hidden_last.astype(mx.float32) | |
| if h.ndim == 1: | |
| h = h[None, :] | |
| if self._slot_w is not None: | |
| logits = h @ self._slot_w.T | |
| else: | |
| # Quantized or wrapped head: use the official last-token linear path, then gather slot rows. | |
| h = hidden_last if hidden_last.ndim > 1 else hidden_last[None] | |
| if hasattr(self.model, "lm_head"): | |
| full = self.model.lm_head(h) | |
| else: | |
| full = self._embed.as_linear(h) | |
| full = full.astype(mx.float32) | |
| logits = full[:, self._slot_ids_mx] | |
| if self.readout.logit_softcap: | |
| logits = _logit_softcap(self.readout.logit_softcap, logits) | |
| return logits | |
| def _forward_hidden(self, token_ids: list[int], cache): | |
| import mlx.core as mx | |
| text = _text_model(self.model) | |
| x = mx.array([token_ids]) | |
| hidden = text(x, cache=cache) | |
| mx.eval(hidden) | |
| return hidden | |
| def decide(self, state: Any, questions: dict[str, Any], debug: bool = False) -> dict[str, Any]: | |
| import mlx.core as mx | |
| t0 = time.perf_counter() | |
| rendered = [render_question(qid, q, style=self.style) for qid, q in questions.items()] | |
| prefix_ids, suffixes, prompts = prefix_and_suffix_ids( | |
| self.tokenizer, state, rendered, self.style, self.template_kwargs | |
| ) | |
| t1 = time.perf_counter() | |
| base = self.model.make_cache() | |
| self._forward_hidden(prefix_ids, base) | |
| mx.eval([c.state for c in base if getattr(c, "state", None) is not None]) | |
| answers: dict[str, Any] = {} | |
| diag: dict[str, Any] = {} | |
| for r, suf in zip(rendered, suffixes): | |
| cache = copy.deepcopy(base) | |
| hidden = self._forward_hidden(suf, cache) | |
| logits = self._slot_logits(hidden[0, -1]) | |
| mx.eval(logits) | |
| slot = logits[0, : r.n_slots].tolist() | |
| probs = softmax(slot, self.calibration.temperature(r.kind, r.n_slots)) | |
| if r.kind == "noul": | |
| answers[r.qid] = make_noul(probs[0]) | |
| elif r.kind == "choice": | |
| answers[r.qid] = make_choice(r.labels, probs) | |
| else: | |
| answers[r.qid] = make_score(r.descriptions, probs) | |
| if debug: | |
| diag[r.qid] = {"slot_logits": slot, "temperature": self.calibration.temperature(r.kind, r.n_slots)} | |
| del cache, hidden | |
| t2 = time.perf_counter() | |
| total = len(prefix_ids) + sum(len(s) for s in suffixes) | |
| out = { | |
| "answers": answers, | |
| "usage": {"input_tokens": total, "output_tokens": 0}, | |
| "qev": { | |
| "prompt_version": self.style.name, | |
| "prefix_tokens": len(prefix_ids), | |
| "branches": len(rendered), | |
| "attention": "blocks" if self.readout.block_bidirectional else "causal", | |
| "readout": self.readout.readout, | |
| "timing": {"render_s": t1 - t0, "forward_s": t2 - t1, "total_s": time.perf_counter() - t0}, | |
| }, | |
| } | |
| if debug: | |
| out["qev"]["diagnostics"] = diag | |
| out["qev"]["prompt"] = prompts[0] | |
| return out | |