clairic-4bit / clairic_runtime /mlx_engine.py
lucataco's picture
Clairic, MLX 4-bit, with runtime, server and conversion script
1a48d6b verified
Raw History Blame Contribute Delete
53.5 kB
"""Clairic inference on Apple silicon with MLX.
Not part of the Cloudflare release: this module replaces its PyTorch engine (`native.py`, `math.py`,
`readout.py`, the typed head in `model.py` and the CUDA kernels). It keeps the same two entry points the request
pipeline calls, with the same inputs and outputs:
- `read_typed(item, media)`: one forward pass of the backbone over the typed-field prompt; the final hidden
states feed the typed head (dates, times, durations, computed numbers, estimates, forecasts);
- `async_generate(...)`: greedy decoding constrained by an xgrammar grammar (JSON schema, regex or EBNF), one
prompt per field or list item. Token log-probabilities are taken after the grammar mask, as upstream does.
The backbone is mlx-vlm's `qwen3_5` model (causal full attention every fourth layer, Gated DeltaNet linear
attention elsewhere) run through an explicit layer loop with a key/value cache for the attention layers and a
recurrent state plus convolution window for the linear layers.
"""
from __future__ import annotations
import asyncio
import concurrent.futures
import hashlib
import json
import threading
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_vlm.models.qwen3_5 import Model, ModelConfig
from mlx_vlm.models.qwen3_5.gated_delta import gated_delta_update
from . import model as cm
ATTENTION_ELEMENTS = 1 << 27 # attention scores per call (batch x heads x queries x keys); bounds temporaries
SEGMENT_TOKENS = 1024 # tokens (rows x length) run through the layers at once; longer inputs go in segments
MEMORY_FRACTION = 0.75 # default ceiling on MLX memory, as a fraction of the machine's RAM
ROPE_MODES = ("mrope", "temporal")
class PreciseRMSNorm(nn.Module):
"""RMSNorm computed in fp32 and cast back, as the reference does. The weight is stored as (1 + w) in fp32."""
def __init__(self, dims: int, eps: float) -> None:
super().__init__()
self.weight = mx.ones((dims,))
self.eps = eps
def __call__(self, x: mx.array) -> mx.array:
return mx.fast.rms_norm(x.astype(mx.float32), self.weight.astype(mx.float32), self.eps).astype(x.dtype)
@dataclass
class Cache:
"""Per-layer state after a prefix: attention keys/values, or the linear layers' memory and conv window."""
keys: list[mx.array | None]
values: list[mx.array | None]
memory: list[mx.array | None]
window: list[mx.array | None]
valid: mx.array | None = None # [B, T] False on padding; None when no row is padded
length: int = 0
position: mx.array | None = None # [B] rotary position of the next token
@classmethod
def empty(cls, layers: int) -> Cache:
return cls([None] * layers, [None] * layers, [None] * layers, [None] * layers)
def arrays(self) -> list[mx.array]:
return [a for group in (self.keys, self.values, self.memory, self.window) for a in group if a is not None]
# ---------------------------------------------------------------------------------------------- typed head
def _normalize(x: mx.array) -> mx.array:
return x / mx.maximum(mx.sqrt((x * x).sum(axis=-1, keepdims=True)), 1e-12)
def _span_scores(start: mx.array, end: mx.array, length_bias: mx.array) -> mx.array:
"""scores[q, s, l] = start[q, s] + end[q, s + l] + bias[l]; spans running past the input are -inf."""
width, tokens = length_bias.shape[0], end.shape[1]
padded = mx.concatenate([end, mx.full((end.shape[0], width - 1), -mx.inf, dtype=end.dtype)], axis=1)
index = mx.arange(tokens)[:, None] + mx.arange(width)[None, :]
return start[:, :, None] + padded[:, index] + length_bias
def _two_digits(slots: mx.array, values: range) -> mx.array:
tens = mx.array([value // 10 for value in values])
ones = mx.array([value % 10 for value in values])
return slots[:, 0, :][:, tens] + slots[:, 1, :][:, ones]
def _years(slots: mx.array) -> mx.array:
index = [[int(f"{year:04d}"[position]) for year in range(cm.YEAR_LOW, cm.YEAR_HIGH + 1)]
for position in range(cm.YEAR_DIGITS)]
inside = sum(slots[:, position, :][:, mx.array(index[position])] for position in range(cm.YEAR_DIGITS))
total = mx.logsumexp(slots, axis=-1).sum(axis=-1)
gap = mx.minimum(mx.logsumexp(inside, axis=-1) - total, -1e-06)
outside = total + mx.log(-mx.expm1(gap))
return mx.concatenate([inside, outside[:, None]], axis=-1)
class TypedHead:
"""The Clairic typed head (`clair_head.safetensors`) with the lexical readout, evaluated in fp32.
A port of upstream `ClairicHead.forward_lexical` and `answer_slot_states`: field queries are routed through
two pre-norm transformer decoder layers that cross-attend the prompt, then read out as span pointers,
digit / month / sign slots, estimate bins and list-item states.
"""
REQUIRED = {"readout": "lexical", "estimate_lexical": True, "span_trim": True, "answer_slots": True,
"answer_slot_routing": True, "number_groups": True, "number_pointer": True}
def __init__(self, path: Path, round_bf16: bool) -> None:
self.config = json.loads((path / "clair_head_config.json").read_text())
for name, value in self.REQUIRED.items():
if self.config.get(name) != value:
raise ValueError(f"unsupported typed head: {name} must be {value!r}")
if self.config.get("loop_feedback"):
raise ValueError("unsupported typed head: loop_feedback")
weights = mx.load(str(path / "clair_head.safetensors"))
if round_bf16:
# Upstream serving builds the head in bf16 before widening it to fp32 (the length bias excepted).
weights = {name: value if name == "length_bias" else value.astype(mx.bfloat16)
for name, value in weights.items()}
self.w = {name: value.astype(mx.float32) for name, value in weights.items()}
mx.eval(self.w)
self.bins = self.w["bins.weight"].shape[0]
self.pointer_width = int(self.config["pointer_width"])
self.heads = int(self.config["route_heads"])
self.layers = int(self.config["route_layers"])
self.slot_rank = int(self.config["slot_rank"])
def eval(self) -> TypedHead:
return self
# ---- building blocks
def linear(self, name: str, x: mx.array) -> mx.array:
y = x @ self.w[f"{name}.weight"].T
bias = self.w.get(f"{name}.bias")
return y if bias is None else y + bias
def norm(self, name: str, x: mx.array) -> mx.array:
return mx.fast.layer_norm(x, self.w[f"{name}.weight"], self.w[f"{name}.bias"], 1e-5)
def _project(self, prefix: str, part: int, x: mx.array) -> mx.array:
weight, bias = self.w[f"{prefix}.in_proj_weight"], self.w[f"{prefix}.in_proj_bias"]
width = weight.shape[1]
y = x @ weight[part * width:(part + 1) * width].T + bias[part * width:(part + 1) * width]
return y.reshape(*x.shape[:-1], self.heads, -1).transpose(0, 2, 1, 3)
def _attention(self, prefix: str, queries: mx.array, keys: mx.array, values: mx.array,
mask: mx.array | None) -> mx.array:
batch = queries.shape[0]
if keys.shape[0] != batch:
keys = mx.broadcast_to(keys, (batch, *keys.shape[1:]))
values = mx.broadcast_to(values, (batch, *values.shape[1:]))
out = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=queries.shape[-1] ** -0.5, mask=mask)
out = out.transpose(0, 2, 1, 3).reshape(batch, queries.shape[2], -1)
return self.linear(f"{prefix}.out_proj", out)
def _memory(self, hidden: mx.array, stop: int) -> list[tuple[mx.array, mx.array]]:
"""The prompt as cross-attention memory: each route layer's keys and values, computed once per read."""
memory = self.linear("memory_in", self.norm("token_norm", hidden[:stop]))[None]
projected = []
for layer in range(self.layers):
prefix = f"route_layers.{layer}.multihead_attn"
projected.append((self._project(prefix, 1, memory), self._project(prefix, 2, memory)))
mx.eval(projected)
return projected
def _route_layers(self, queries: mx.array, memory: list[tuple[mx.array, mx.array]],
padding: mx.array | None = None) -> mx.array:
"""`nn.TransformerDecoderLayer(norm_first=True, activation="gelu")` stack; `padding` marks ignored keys."""
mask = None if padding is None else mx.logical_not(padding)[:, None, None, :]
for layer, (keys, values) in enumerate(memory):
prefix = f"route_layers.{layer}"
x = self.norm(f"{prefix}.norm1", queries)
queries = queries + self._attention(
f"{prefix}.self_attn", self._project(f"{prefix}.self_attn", 0, x),
self._project(f"{prefix}.self_attn", 1, x), self._project(f"{prefix}.self_attn", 2, x), mask)
x = self.norm(f"{prefix}.norm2", queries)
queries = queries + self._attention(
f"{prefix}.multihead_attn", self._project(f"{prefix}.multihead_attn", 0, x), keys, values, None)
x = self.norm(f"{prefix}.norm3", queries)
queries = queries + self.linear(f"{prefix}.linear2", nn.gelu(self.linear(f"{prefix}.linear1", x)))
return queries
# ---- readouts
def typed_logits(self, raw: mx.array, routed: mx.array) -> tuple[mx.array, mx.array, mx.array]:
count, slots = raw.shape[0], cm.DIGIT_SLOTS
scale = mx.exp(mx.minimum(self.w["lexical_scale"], float(np.log(100.0))))
unit = _normalize(raw)
down = (raw @ self.w["slot_down.weight"].T).reshape(count, slots + 2, self.slot_rank)
digit_prior = (unit @ _normalize(self.w["digit_rows"]).T)[:, None, :]
month_prior = unit @ _normalize(self.w["month_rows"]).T
sign_prior = unit @ _normalize(self.w["sign_rows"]).T
residual = mx.sigmoid(self.w["residual_gate"]) * self.linear("slot_residual", routed)
digit_residual = residual[:, : slots * 10].reshape(count, slots, 10)
month_residual, sign_residual = residual[:, slots * 10: slots * 10 + 12], residual[:, slots * 10 + 12:]
return (
scale[0] * digit_prior + mx.einsum("nsr,srd->nsd", down[:, :slots], self.w["slot_up"]) + digit_residual,
scale[1] * month_prior + down[:, slots] @ self.w["month_up"] + month_residual,
scale[2] * sign_prior + down[:, slots + 1] @ self.w["sign_up"] + sign_residual,
)
def typed_outputs(self, raw: mx.array, states: mx.array, routed: mx.array) -> dict[str, mx.array]:
digits, months, signs = self.typed_logits(raw, routed)
count = raw.shape[0]
year_slots = digits[:, cm.YEAR_SLOT: cm.YEAR_SLOT + cm.YEAR_DIGITS]
return {
"number": mx.concatenate([signs, self.linear("exponent", states),
digits[:, : cm.NUMBER_DIGITS].reshape(count, -1)], axis=-1),
"date": mx.concatenate([year_slots.reshape(count, -1), _years(year_slots), months,
_two_digits(digits[:, cm.DAY_SLOT: cm.DAY_SLOT + 2], range(1, 32))], axis=-1),
"time": mx.concatenate([_two_digits(digits[:, cm.HOUR_SLOT: cm.HOUR_SLOT + 2], range(24)),
_two_digits(digits[:, cm.MINUTE_SLOT: cm.MINUTE_SLOT + 2], range(60)),
_two_digits(digits[:, cm.SECOND_SLOT: cm.SECOND_SLOT + 2], range(60))], axis=-1),
}
def group_scores(self, states: mx.array) -> mx.array:
return self.linear("number_group_up", nn.gelu(self.linear("number_group_down", states))).reshape(
-1, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)
def temper_number(self, logits: mx.array, spread: mx.array) -> mx.array:
classes, digits = cm.EXPONENT_CLASSES, cm.NUMBER_DIGITS
sign, exponent = logits[:, :2], logits[:, 2: 2 + classes]
slots = logits[:, 2 + classes:].reshape(-1, digits, 10)
limit = cm.ESTIMATE_SPREAD_LIMIT
temperature = mx.exp(limit * mx.tanh(spread / limit))
slots = mx.concatenate([slots[:, : cm.LEXICAL_DIGITS] / temperature[:, 1:, None], slots[:, cm.LEXICAL_DIGITS:]], axis=1)
return mx.concatenate([sign, exponent / temperature[:, :1], slots.reshape(slots.shape[0], -1)], axis=-1)
def forward(self, hidden: mx.array, start: int, stop: int, views: dict[str, np.ndarray],
memory: list[tuple[mx.array, mx.array]]) -> dict[str, mx.array]:
"""Upstream `forward_lexical`: `hidden` is [tokens, width] in fp32, `views` the index arrays."""
if any(name.endswith("_prefix_length") for name in views):
raise ValueError("prefixed number readouts are not supported")
rows = {name: np.asarray(value) for name, value in views.items()}
queries = rows["query_positions"]
item_kinds = [kind for kind in cm.LIST_KINDS if rows[f"{kind}_rows"].shape[0]]
positions = np.concatenate([queries] + [rows[f"{kind}_items"].reshape(-1) for kind in item_kinds])
kinds = np.zeros(positions.shape[0], dtype=np.int64)
for index, group in enumerate(cm.ROUTE_KINDS):
kinds[rows[f"{group}_rows"]] = index
offset = queries.shape[0]
for kind in item_kinds:
size = rows[f"{kind}_items"].size
kinds[offset: offset + size] = len(cm.ROUTE_KINDS) + cm.LIST_KINDS.index(kind)
offset += size
def take(values: mx.array, index: np.ndarray) -> mx.array:
return values[mx.array(index.astype(np.int32))]
raw = take(hidden, positions)
routed = self._route_layers(
(self.linear("route_in", self.norm("norm", raw)) + take(self.w["kind_embedding.weight"], kinds))[None], memory)
routed = self.norm("route_norm", routed[0])
states = self.norm("norm", raw) + mx.sigmoid(self.w["route_gate"]) * self.linear("route_out", routed)
count = queries.shape[0]
output: dict[str, mx.array] = {"null": self.linear("null", states[:count])[:, 0]}
field_states, field_raw, field_routed = states[:count], raw[:count], routed[:count]
text, tags, numbers = rows["text_rows"], rows["tags_rows"], rows["number_rows"]
scale = 1.0 / float(np.sqrt(self.pointer_width))
tokens = None
if text.shape[0] or tags.shape[0] or numbers.shape[0]:
tokens = self.norm("token_norm", hidden[start:stop])
if text.shape[0] or tags.shape[0]:
output["trim"] = self.linear("trim", tokens).reshape(-1, 2, cm.TRIM_CLASSES)
def pointer(query: str, selected: mx.array) -> mx.array:
query_start, query_end = mx.split(self.linear(query, selected), 2, axis=-1)
key_start, key_end = mx.split(self.linear("span_key", tokens), 2, axis=-1)
return _span_scores(query_start @ key_start.T * scale, query_end @ key_end.T * scale, self.w["length_bias"])
if numbers.shape[0]:
number_states = take(field_states, numbers)
output["number_copy"] = pointer("copy_query", number_states)
output["number_copy_gate"] = self.linear("copy_gate", number_states)[:, 0]
if text.shape[0]:
output["text"] = pointer("span_query", take(field_states, text))
if tags.shape[0]:
query_begin, query_inside = mx.split(self.linear("tag_query", take(field_states, tags)), 2, axis=-1)
keys = self.linear("tag_key", tokens)
begin = query_begin @ keys.T * scale + self.w["tag_bias"][0]
inside = query_inside @ keys.T * scale + self.w["tag_bias"][1]
output["tags"] = mx.stack([mx.zeros_like(begin), begin, inside], axis=-1)
typed_rows = [rows[f"{group}_rows"] for group in ("number", "date", "time")]
if any(selected.shape[0] for selected in typed_rows):
selected = np.concatenate(typed_rows)
typed = self.typed_outputs(take(field_raw, selected), take(field_states, selected), take(field_routed, selected))
offset = 0
for group, selected in zip(("number", "date", "time"), typed_rows):
if selected.shape[0]:
output[group] = typed[group][offset: offset + selected.shape[0]]
offset += selected.shape[0]
if "number" in output:
output["number_groups"] = self.group_scores(take(field_states, numbers))
estimates = rows["estimate_rows"]
if estimates.shape[0]:
estimate_states = take(field_states, estimates)
output["estimate"] = self.linear("bins", estimate_states)
output["estimate_offset"] = self.linear("offset", estimate_states)
output["estimate_overflow"] = self.linear("overflow", estimate_states)
number_logits = self.typed_outputs(take(field_raw, estimates), estimate_states, take(field_routed, estimates))["number"]
output["estimate_number_raw"] = number_logits
output["estimate_number"] = self.temper_number(number_logits, self.linear("estimate_spread", estimate_states))
output["estimate_gate"] = self.linear("estimate_gate", estimate_states)[:, 0]
offset = count
for kind in item_kinds:
shape = rows[f"{kind}_items"].shape
size = shape[0] * shape[1]
item_states, item_raw, item_routed = (
states[offset: offset + size], raw[offset: offset + size], routed[offset: offset + size])
offset += size
output[f"{kind}_end"] = self.linear("item_end", item_states)[:, 0].reshape(shape)
group = "number" if kind == "number_list" else "date"
output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[group].reshape(*shape, -1)
if kind == "number_list":
output["number_list_groups"] = self.group_scores(item_states).reshape(
*shape, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)
return output
def answer_slot_states(self, hidden: mx.array, field_queries: np.ndarray, kinds: np.ndarray, positions: np.ndarray,
lengths: np.ndarray, stop: int, memory: list[tuple[mx.array, mx.array]],
chunk: int = 256) -> mx.array:
"""Upstream `answer_slot_states`: each answer-slot position corrected by evidence routed from the prompt."""
width = positions.shape[1]
if (width < 1 or positions.shape[0] != field_queries.shape[0] or (lengths < 1).any() or (lengths > width).any()
or not 0 < stop <= hidden.shape[0]):
raise ValueError("invalid answer-slot routing layout")
gate = mx.sigmoid(self.w["route_gate"])
output = []
field_batch = max(1, min(chunk // (width + 1), 262144 // stop))
for first in range(0, positions.shape[0], field_batch):
last = first + field_batch
slot_positions = mx.array(positions[first:last].astype(np.int32))
query_positions = mx.concatenate([mx.array(field_queries[first:last, None].astype(np.int32)), slot_positions], axis=1)
row_lengths = mx.array(lengths[first:last].astype(np.int32))
row_kinds = mx.array(kinds[first:last].astype(np.int32))
padding = mx.arange(width + 1)[None, :] > row_lengths[:, None]
queries = self.linear("route_in", self.norm("norm", hidden[query_positions]))
queries = self._route_layers(queries + self.w["kind_embedding.weight"][row_kinds][:, None, :], memory, padding)
evidence = self.linear("route_out", self.norm("route_norm", queries[:, 1:]))
scale = mx.tanh(self.w["slot_route_scale"][row_kinds])[:, None, None]
corrected = hidden[slot_positions] + scale * gate * evidence
index = mx.minimum(mx.arange(width)[None, :], row_lengths[:, None] - 1)
output.append(mx.take_along_axis(corrected, index[:, :, None], axis=1))
mx.eval(output[-1])
return mx.concatenate(output, axis=0)
# ---------------------------------------------------------------------------------------------- engine
@dataclass
class _Rows:
"""One right-padded batch: token ids, which positions are real, and the rows' image inputs in order."""
ids: np.ndarray
valid: np.ndarray
media: list[dict[str, Any]] = field(default_factory=list)
@property
def lengths(self) -> np.ndarray:
return self.valid.sum(axis=1)
def _common_prefix(sequences: list[list[int]]) -> int:
first, last = min(sequences), max(sequences)
return next((i for i, (a, b) in enumerate(zip(first, last)) if a != b), min(len(first), len(last)))
class MLXEngine:
"""Share one MLX backbone between grammar-constrained extraction and typed readouts."""
def __init__(self, path: Path, tokenizer: Any, *, batch_size: int = 8, batch_token_budget: int = 8192,
rope: str = "temporal", device: str = "gpu", memory_limit_gb: float | None = None) -> None:
import xgrammar as xgr
if rope not in ROPE_MODES:
raise ValueError(f"rope must be one of {ROPE_MODES}")
self.path, self.tokenizer, self.rope = Path(path), tokenizer, rope
self.batch_size, self.batch_token_budget = max(1, int(batch_size)), max(1, int(batch_token_budget))
self.text_batch_size = self.batch_size
self.device = device # "gpu", or "cpu" for bit-accurate float32 checks (slow)
self.capture: Callable[[str, Any], None] | None = None # test hook: receives intermediate arrays
# A request that would need more memory than this fails with an error instead of exhausting the machine.
self.memory_cap = int((memory_limit_gb * 1e9) if memory_limit_gb else MEMORY_FRACTION * mx.device_info()["memory_size"])
self._worker = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="clairic-model")
self._features: dict[str, mx.array] = {}
self._prefixes: dict[tuple, Cache] = {}
self.prefix_cache = True # read a request's images once and continue every prompt from that state
# Every MLX call happens on the worker thread, including loading the weights.
self._worker.submit(self._load).result()
self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
self.pad = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else self.eos
self.vocab = self.lm.lm_head.weight.shape[0]
info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=self.vocab, stop_token_ids=[self.eos])
self.compiler = xgr.GrammarCompiler(info, max_threads=2)
self._xgr = xgr
# ---- loading
def _load(self) -> None:
path = self.path
if self.device == "cpu":
mx.set_default_device(mx.cpu)
config = json.loads((path / "config.json").read_text())
self.model = Model(ModelConfig.from_dict(config))
self.lm = self.model.language_model
text = self.lm.model
hidden, eps = config["text_config"]["hidden_size"], config["text_config"]["rms_norm_eps"]
text.norm = PreciseRMSNorm(hidden, eps)
for layer in text.layers:
layer.input_layernorm = PreciseRMSNorm(hidden, eps)
layer.post_attention_layernorm = PreciseRMSNorm(hidden, eps)
if not layer.is_linear:
layer.self_attn.q_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
layer.self_attn.k_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
weights: dict[str, mx.array] = {}
for shard in sorted(path.glob("model*.safetensors")):
weights.update(mx.load(str(shard)))
if quantization := config.get("quantization"):
def quantized(path: str, module: nn.Module) -> bool | dict:
if not hasattr(module, "to_quantized") or f"{path}.scales" not in weights:
return False
settings = quantization.get(path) # modules with their own bit width
return settings if isinstance(settings, dict) else True
nn.quantize(self.model, group_size=quantization["group_size"], bits=quantization["bits"],
mode=quantization.get("mode", "affine"), class_predicate=quantized)
self.model.load_weights(list(weights.items()), strict=True)
self.model.eval()
mx.eval(self.model.parameters())
self.image_token = int(config["image_token_id"])
self.vision_end = int(config["vision_end_token_id"])
self.dtype = text.layers[0].linear_attn.conv1d.weight.dtype # never quantized: bf16 (release) or fp32
self.backbone = self.model
self.head = TypedHead(path, round_bf16=self.dtype != mx.float32)
# ---- request scope (image features are kept for one request)
def begin_request(self) -> None:
self._features, self._prefixes = {}, {}
async def end_request(self) -> None:
await asyncio.get_running_loop().run_in_executor(self._worker, self._clear)
def _clear(self) -> None:
self._features, self._prefixes = {}, {}
mx.clear_cache()
def shutdown(self) -> None:
self._worker.shutdown(wait=True, cancel_futures=True)
self._features, self._prefixes = {}, {}
def _check_memory(self) -> None:
if mx.get_active_memory() > self.memory_cap:
mx.clear_cache()
if mx.get_active_memory() > self.memory_cap:
raise MemoryError(
f"this request needs more than the {self.memory_cap / 1e9:.0f} GB memory limit; "
"shorten the input, lower batch_token_budget, or raise memory_limit_gb")
# ---- backbone
def _image_features(self, mapping: dict[str, Any]) -> mx.array:
key = mapping.get("clair_pixels_key")
if key is None:
pixels = np.asarray(mapping["pixel_values"])
key = hashlib.sha1(pixels.tobytes() + np.asarray(mapping["image_grid_thw"]).tobytes()).hexdigest()
if key not in self._features:
tower = self.model.vision_tower
pixels = mx.array(mapping["pixel_values"]).astype(tower.patch_embed.proj.weight.dtype)
features, _ = tower(pixels, mx.array(np.asarray(mapping["image_grid_thw"])))
mx.eval(features)
self._features[key] = features
return self._features[key]
def _rows(self, sequences: list[list[int]], media: list[Any]) -> _Rows:
width = max(map(len, sequences))
ids = np.full((len(sequences), width), self.pad, dtype=np.int32)
valid = np.zeros((len(sequences), width), dtype=bool)
for row, sequence in enumerate(sequences):
ids[row, : len(sequence)] = sequence
valid[row, : len(sequence)] = True
return _Rows(ids, valid, [mapping for payload in media if payload for mapping in payload])
def _embed(self, rows: _Rows) -> tuple[mx.array, mx.array]:
"""Input embeddings (image features merged in) and rotary positions [3, B, L] of a right-padded batch."""
tokens, mask = mx.array(rows.ids), mx.array(rows.valid.astype(np.int32))
if rows.media:
features = mx.concatenate([self._image_features(mapping) for mapping in rows.media], axis=0)
grids = mx.array(np.concatenate([np.asarray(mapping["image_grid_thw"]) for mapping in rows.media]))
embedded = self.model.get_input_embeddings(
tokens, features, image_grid_thw=grids, mask=mask, cached_image_features=features)
else:
embedded = self.model.get_input_embeddings(tokens, None, mask=mask)
positions = embedded.position_ids
if positions.ndim == 2: # text only: the same position on the three rotary axes
positions = mx.broadcast_to(positions[None], (3, *positions.shape))
return embedded.inputs_embeds, positions
def _rotary(self, positions: mx.array) -> mx.array:
# "temporal" (default) is what the upstream runtime does in bfloat16, its serving configuration: the
# rotary table is indexed by the first (temporal) position axis for every frequency, so the tokens of an
# image share one rotary position. "mrope" is the stock Transformers 3-axis rotary, which upstream runs
# in float32. Text tokens are identical in both.
if self.rope == "temporal":
return mx.broadcast_to(positions[:1], positions.shape)
return positions
def _linear(self, attention: nn.Module, x: mx.array, valid: mx.array | None, window: mx.array | None,
memory: mx.array | None, lengths: mx.array | None = None) -> tuple[mx.array, mx.array, mx.array]:
"""A Gated DeltaNet layer continuing from a convolution window and recurrent memory (padding skipped)."""
batch, length, _ = x.shape
mixed = attention.in_proj_qkv(x)
z = attention.in_proj_z(x).reshape(batch, length, -1, attention.head_v_dim)
b, a = attention.in_proj_b(x), attention.in_proj_a(x)
if valid is not None:
mixed = mx.where(valid[..., None], mixed, 0)
if window is None:
window = mx.zeros((batch, attention.conv_kernel_size - 1, attention.conv_dim), dtype=mixed.dtype)
conv_input = mx.concatenate([window.astype(mixed.dtype), mixed], axis=1)
conv = nn.silu(attention.conv1d(conv_input))
q, k, v = (
t.reshape(batch, length, heads, width)
for t, heads, width in zip(
mx.split(conv, [attention.key_dim, 2 * attention.key_dim], -1),
[attention.num_k_heads, attention.num_k_heads, attention.num_v_heads],
[attention.head_k_dim, attention.head_k_dim, attention.head_v_dim], strict=True)
)
q, k = attention._normalize_qk(q, k)
out, memory = gated_delta_update(q, k, v, a, b, attention.A_log, attention.dt_bias, state=memory, mask=valid,
use_kernel=True)
out = attention.out_proj(attention.norm(out, z).reshape(batch, length, -1))
taps = attention.conv_kernel_size - 1
if lengths is None:
return out, memory, conv_input[:, -taps:]
# Right-padded rows: keep the last real inputs of each row, not the padding after them.
index = lengths[:, None] + mx.arange(taps)[None, :]
index = mx.broadcast_to(index[:, :, None], (batch, taps, conv_input.shape[-1]))
return out, memory, mx.take_along_axis(conv_input, index, axis=1)
def _attention(self, attention: nn.Module, x: mx.array, positions: mx.array, keys: mx.array | None,
values: mx.array | None, valid: mx.array | None, offset: int) -> tuple[mx.array, mx.array, mx.array]:
"""Causal attention of new tokens over the cached and new keys; padded keys are masked."""
batch, length, _ = x.shape
queries, new_keys, new_values, gate, _ = attention._prepare_projected_qkv(
attention.q_proj(x), attention.k_proj(x), attention.v_proj(x), None, positions, None, None)
if keys is not None:
new_keys = mx.concatenate([keys, new_keys], axis=2)
new_values = mx.concatenate([values, new_values], axis=2)
total = new_keys.shape[2]
# The score matrix is queries x keys per head. Long inputs are attended in slices of query rows so its
# size stays bounded (a slice's output depends only on its own rows); unsliced, a 40k-token input needs
# tens of GB per layer.
rows = max(1, ATTENTION_ELEMENTS // (batch * attention.num_attention_heads * total))
outputs = []
for start in range(0, length, rows):
stop = min(length, start + rows)
mask = None
if length > 1:
key_index = mx.arange(total)[None, :]
row_index = mx.arange(offset + start, offset + stop)[:, None]
mask = (key_index <= row_index)[None, None]
if valid is not None: # a padded query still sees itself, so no row is fully masked
mask = mask & (valid[:, None, None, :] | (key_index == row_index)[None, None])
elif valid is not None:
mask = valid[:, None, None, :]
outputs.append(mx.fast.scaled_dot_product_attention(
queries[:, :, start:stop], new_keys, new_values, scale=attention.scale, mask=mask))
if rows < length:
mx.eval(outputs[-1])
self._check_memory()
output = outputs[0] if len(outputs) == 1 else mx.concatenate(outputs, axis=2)
output = output.transpose(0, 2, 1, 3).reshape(batch, length, -1)
return attention.o_proj(output * mx.sigmoid(gate)), new_keys, new_values
def _forward(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
keep: bool = True) -> tuple[mx.array, Cache | None]:
"""Run the layers over new tokens, optionally continuing from a cache.
`lengths` gives each row's number of real tokens when the batch is right-padded (None: no padding).
Returns the hidden states before the final norm and, with `keep`, the cache after each row's last real
token (padded positions stay in the attention cache, marked invalid).
"""
layers = self.lm.model.layers
cache = cache or Cache.empty(len(layers))
batch, length, _ = h.shape
valid = row_lengths = None
if lengths is not None and (lengths < length).any():
row_lengths = mx.array(lengths.astype(np.int32))
valid = mx.arange(length)[None, :] < row_lengths[:, None]
all_valid = None
if cache.valid is not None or valid is not None:
before = cache.valid if cache.valid is not None else mx.ones((batch, cache.length), dtype=mx.bool_)
new = valid if valid is not None else mx.ones((batch, length), dtype=mx.bool_)
all_valid = mx.concatenate([before, new], axis=1)
rotary = self._rotary(positions)
updated = Cache.empty(len(layers)) if keep else None
stepwise = batch * length > 4096 # keep the lazy graph small on long inputs
for index, layer in enumerate(layers):
x = layer.input_layernorm(h)
if layer.is_linear:
r, memory, window = self._linear(layer.linear_attn, x, valid, cache.window[index], cache.memory[index],
row_lengths)
if keep:
updated.memory[index], updated.window[index] = memory, window
else:
r, keys, values = self._attention(layer.self_attn, x, rotary, cache.keys[index], cache.values[index],
all_valid, cache.length)
if keep:
updated.keys[index], updated.values[index] = keys, values
h = h + r
h = h + layer.mlp(layer.post_attention_layernorm(h))
if stepwise:
mx.eval(h, *(updated.arrays() if keep else []))
self._check_memory()
if keep:
updated.valid, updated.length = all_valid, cache.length + length
return h, updated
def _run(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
keep: bool = True) -> tuple[mx.array, Cache | None]:
"""`_forward` over a whole input. Long inputs go through the layers in segments, each continuing from
the cache of the previous one, so working memory depends on the segment size rather than the input
length (attention is causal and the linear layers are recurrent, so the result is the same). With
`keep`, the returned cache has `position` set to each row's next rotary position."""
batch, length, _ = h.shape
segment = max(256, SEGMENT_TOKENS // batch)
if length <= segment:
out, cache = self._forward(h, positions, lengths, cache, keep)
else:
outputs = []
for start in range(0, length, segment):
stop = min(length, start + segment)
inside = None if lengths is None else np.clip(lengths - start, 0, stop - start)
part, cache = self._forward(h[:, start:stop], positions[:, :, start:stop], inside, cache)
mx.eval(part, *cache.arrays())
self._check_memory()
outputs.append(part)
out = mx.concatenate(outputs, axis=1)
cache = cache if keep else None
if keep:
last = mx.array((lengths if lengths is not None else np.full(batch, length)).astype(np.int32)) - 1
index = mx.broadcast_to(last[None, :, None], (positions.shape[0], batch, 1))
cache.position = mx.take_along_axis(positions, index, axis=2)[:, :, 0].max(axis=0) + 1
return out, cache
def _hidden(self, sequences: list[list[int]], media: list[Any]) -> mx.array:
"""Final-norm hidden states of an unpadded single sequence."""
rows = self._rows(sequences, media)
h, positions = self._embed(rows)
h, _ = self._run(h, positions, None, None, keep=False)
return self.lm.model.norm(h)
# ---- typed readout (upstream `read_record`)
def _read_record(self, states: mx.array, item: Any) -> dict[str, np.ndarray]:
start, length, kinds, queries, items, slot_fields, slots = cm.parse_prediction_layout(
cm.prediction_layout(item), len(item.input_ids), with_slots=True)
views = cm.prediction_views(kinds, queries, items, None)
hidden = states.astype(mx.float32)
stop = start + length
memory = self.head._memory(hidden, stop)
output = self.head.forward(hidden, start, stop, views, memory)
mx.eval(output)
values: dict[str, np.ndarray] = {name: np.array(value) for name, value in output.items()}
if slots:
width = max(map(len, slots))
positions = np.array([row + [row[-1]] * (width - len(row)) for row in slots], dtype=np.int64)
field_queries = np.asarray(views["query_positions"])[slot_fields]
slot_kinds = np.array([cm.ROUTE_KINDS.index("tags" if kinds[row] == "text_list" else kinds[row])
for row in slot_fields], dtype=np.int64)
lengths = np.array([len(row) for row in slots], dtype=np.int64)
slot_states = self.head.answer_slot_states(hidden, field_queries, slot_kinds, positions, lengths, stop, memory)
slot_states = slot_states.reshape(-1, slot_states.shape[-1]).astype(states.dtype)
ids, logprobs = [], []
for first in range(0, slot_states.shape[0], 256):
logits = self.lm.lm_head(slot_states[first: first + 256]).astype(mx.float32)
best = mx.argmax(logits, axis=-1)
ids.append(best)
logprobs.append(mx.take_along_axis(logits, best[:, None], axis=-1)[:, 0] - mx.logsumexp(logits, axis=-1))
mx.eval(ids[-1], logprobs[-1])
values["slot_ids"] = np.array(mx.concatenate(ids)).reshape(positions.shape)
values["slot_log_probs"] = np.array(mx.concatenate(logprobs)).reshape(positions.shape)
return values
def _read_typed(self, item: Any, media: Any, _cancelled: threading.Event | None = None) -> dict[str, dict[str, Any]]:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
states = self._hidden([list(item.input_ids)], [media])[0]
mx.eval(states)
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
values = self._read_record(states, item)
compact = cm.compact_output(values)
if self.capture is not None:
self.capture("hidden", np.array(states.astype(mx.float32)))
self.capture("compact", np.asarray(compact, dtype=np.float32))
if "slot_ids" in values:
self.capture("slots", values["slot_ids"])
if media:
self.capture("pixels", np.asarray(media[0]["pixel_values"]).astype(np.float16))
self.capture("grid", np.asarray(media[0]["image_grid_thw"]))
values = cm.expand_output(compact, item, bins=self.head.bins, estimate_lexical=True, span_trim=True,
number_groups=True, number_pointer=True)
return cm.decode_record(values, item, self.tokenizer)
async def read_typed(self, item: Any, media: Any = None) -> dict[str, dict[str, Any]]:
return await self._submit(lambda cancelled: self._read_typed(item, media, _cancelled=cancelled))
# ---- constrained generation
def _grammar(self, parameters: dict[str, Any]) -> Any:
if "json_schema" in parameters:
value = parameters["json_schema"]
return self.compiler.compile_json_schema(json.loads(value) if isinstance(value, str) else value,
any_whitespace=True)
if "regex" in parameters:
return self.compiler.compile_regex(parameters["regex"])
if "ebnf" in parameters:
return self.compiler.compile_grammar(parameters["ebnf"])
raise ValueError("Extraction continuations require a JSON schema, regex, or grammar")
def _logits(self, h: mx.array) -> np.ndarray:
"""Vocabulary logits (fp32) from hidden states [B, 1, width] before the final norm."""
return np.array(self.lm.lm_head(self.lm.model.norm(h))[:, 0].astype(mx.float32))
@staticmethod
def _media_key(payload: Any) -> tuple | None:
return tuple(mapping.get("clair_pixels_key") or id(mapping["pixel_values"]) for mapping in payload) if payload else None
def _image_prefix(self, sequences: list[list[int]], media: list[Any]) -> tuple[Cache | None, int]:
"""The state after the images shared by every prompt of a batch, computed once per request.
Image prompts start with the same image tokens and differ only afterwards. The shared part, up to the
last `<|vision_end|>` common to all rows, is run once and each row continues from its cache; attention
is causal and the linear layers are recurrent, so this equals running every full prompt.
"""
if not self.prefix_cache or not media[0] or any(self._media_key(m) != self._media_key(media[0]) for m in media):
return None, 0
common = sequences[0][: _common_prefix(sequences)]
if self.vision_end not in common:
return None, 0
cut = len(common) - common[::-1].index(self.vision_end)
if any(self.image_token in sequence[cut:] for sequence in sequences):
return None, 0
key = (self._media_key(media[0]), hashlib.sha1(np.asarray(common[:cut], dtype=np.int32).tobytes()).hexdigest())
if key not in self._prefixes:
rows = self._rows([common[:cut]], [media[0]])
h, positions = self._embed(rows)
_, cache = self._run(h, positions, None, None)
mx.eval(cache.position, *cache.arrays())
self._prefixes[key] = cache
return self._prefixes[key], cut
def _prefill(self, sequences: list[list[int]], media: list[Any]) -> tuple[mx.array, Cache]:
"""Run the prompts of one batch. Returns each row's last hidden state [B, 1, width] and the cache."""
size = len(sequences)
prefix, cut = self._image_prefix(sequences, media)
if prefix is None:
rows = self._rows(sequences, media)
h, positions = self._embed(rows)
cache = None
else:
rows = self._rows([sequence[cut:] for sequence in sequences], [None] * size)
h = self.lm.model.embed_tokens(mx.array(rows.ids))
positions = prefix.position[0] + mx.arange(rows.ids.shape[1])
positions = mx.broadcast_to(positions[None, None, :], (3, size, rows.ids.shape[1]))
def spread(array: mx.array | None) -> mx.array | None:
return None if array is None else mx.broadcast_to(array, (size, *array.shape[1:]))
cache = Cache([spread(a) for a in prefix.keys], [spread(a) for a in prefix.values],
[spread(a) for a in prefix.memory], [spread(a) for a in prefix.window], None, prefix.length)
lengths = rows.lengths
h, cache = self._run(h, positions, lengths, cache)
last = mx.array((lengths - 1).astype(np.int32))
index = mx.broadcast_to(last[:, None, None], (size, 1, h.shape[-1]))
return mx.take_along_axis(h, index, axis=1), cache
def _decode(self, sequences: list[list[int]], media: list[Any], grammars: list[Any], maximum: int,
stop_ids: tuple[int, ...], score_ids: list[int], cancelled: threading.Event | None,
) -> list[tuple[list[int], list[float], list[float]]]:
"""Greedy decoding of one batch under per-row grammars. Returns (tokens, log-probabilities, first-step
log-probabilities of `score_ids`) per row; tokens end with the stop token when one was produced."""
size = len(sequences)
h, cache = self._prefill(sequences, media)
mx.eval(h, cache.position, *cache.arrays())
scores = self._logits(h)
matchers = [self._xgr.GrammarMatcher(grammar) for grammar in grammars]
bitmask = np.zeros(self._xgr.get_bitmask_shape(size, self.vocab), dtype=np.int32)
tokens: list[list[int]] = [[] for _ in range(size)]
logprobs: list[list[float]] = [[] for _ in range(size)]
first: list[list[float]] = [[] for _ in range(size)]
done = [False] * size
for step in range(maximum):
for row, matcher in enumerate(matchers):
if not matcher.is_terminated():
matcher.fill_next_token_bitmask(bitmask, row)
allowed = np.unpackbits(bitmask.view(np.uint8), axis=-1, bitorder="little")[:, : self.vocab].astype(bool)
masked = np.where(allowed, scores, -np.inf).astype(np.float32)
peak = masked.max(axis=-1, keepdims=True)
with np.errstate(divide="ignore"):
log_probabilities = masked - peak - np.log(np.exp(masked - peak).sum(axis=-1, keepdims=True, dtype=np.float32))
choices = masked.argmax(axis=-1)
for row in range(size):
if step == 0:
first[row] = [float(log_probabilities[row, token]) for token in score_ids]
if done[row]:
continue
token = int(choices[row])
tokens[row].append(token)
logprobs[row].append(float(log_probabilities[row, token]))
if token in stop_ids:
done[row] = True
if not matchers[row].is_terminated() and not matchers[row].accept_token(token):
raise RuntimeError("Generated token rejected by its grammar")
if all(done) or step + 1 == maximum:
break
if cancelled is not None and cancelled.is_set():
raise RuntimeError("Inference cancelled")
feed = mx.array([[self.pad if done[row] else tokens[row][-1]] for row in range(size)])
position = cache.position
h, cache = self._forward(self.lm.model.embed_tokens(feed), mx.broadcast_to(position[None, :, None], (3, size, 1)),
None, cache)
cache.position = position + 1
mx.eval(h, cache.position, *cache.arrays())
scores = self._logits(h)
return list(zip(tokens, logprobs, first, strict=True))
def _generate(self, *, input_ids: list[int] | list[list[int]], image_data: Any = None,
sampling_params: dict[str, Any] | list[dict[str, Any]], rid: Any = None,
token_ids_logprob: list[int] | None = None, _cancelled: threading.Event | None = None,
**kwargs: Any) -> Any:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
if not input_ids:
return []
single = isinstance(input_ids[0], int)
sequences = [input_ids] if single else input_ids
size = len(sequences)
parameters = [sampling_params] * size if isinstance(sampling_params, dict) else sampling_params
media = [image_data] if single else (image_data or [None] * size)
ids = [rid] if single else (rid or [None] * size)
if not (len(parameters) == len(media) == len(ids) == size):
raise ValueError("Extraction batch arguments have inconsistent lengths")
results: list[Any] = [None] * size
groups: dict[tuple[int, tuple[int, ...]], list[int]] = defaultdict(list)
for index, params in enumerate(parameters):
if params.get("temperature", 0) != 0:
raise ValueError("Clairic extraction uses deterministic decoding")
groups[(int(params.get("max_new_tokens", 256)), tuple(params.get("stop_token_ids", [self.eos])))].append(index)
for (maximum, stop_ids), indices in groups.items():
# Rows with the same images and similar length share a batch; a row's result does not depend on
# its batch. Rows that share images are charged for the image prefix once.
indices = sorted(indices, key=lambda index: (str(self._media_key(media[index])), len(sequences[index])))
chunks: list[list[int]] = []
pending: list[int] = []
for index in indices:
if pending and (len(pending) >= self.batch_size
or self._media_key(media[index]) != self._media_key(media[pending[0]])
or self._cost([sequences[i] for i in (*pending, index)], media[index]) > self.batch_token_budget):
chunks.append(pending)
pending = []
pending.append(index)
if pending:
chunks.append(pending)
for chunk in chunks:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
decoded = self._decode([sequences[i] for i in chunk], [media[i] for i in chunk],
[self._grammar(parameters[i]) for i in chunk], maximum, stop_ids,
list(token_ids_logprob or []), _cancelled)
for index, (output, logprobs, first) in zip(chunk, decoded, strict=True):
stopped = bool(output) and output[-1] in stop_ids
results[index] = {
"output_ids": output,
"text": self.tokenizer.decode(output[:-1] if stopped else output, skip_special_tokens=False,
clean_up_tokenization_spaces=False),
"meta_info": {
"id": ids[index],
"completion_tokens": len(output),
"finish_reason": {"type": "stop", "matched": output[-1]} if stopped else {"type": "length"},
"output_token_logprobs": [(value, token, None) for value, token in zip(logprobs, output, strict=True)],
"output_token_ids_logprobs": [[(value, token, None) for value, token in
zip(first, token_ids_logprob or [], strict=True)]],
},
}
return results[0] if single else results
def _cost(self, sequences: list[list[int]], payload: Any) -> int:
"""Tokens a batch runs through the model: the padded width times its rows (shared image tokens once)."""
shared = 0
if payload and self.prefix_cache and len(sequences) > 1:
common = sequences[0][: _common_prefix(sequences)]
shared = len(common) - common[::-1].index(self.vision_end) if self.vision_end in common else 0
return (max(map(len, sequences)) - shared) * len(sequences)
async def async_generate(self, **kwargs: Any) -> Any:
return await self._submit(lambda cancelled: self._generate(_cancelled=cancelled, **kwargs))
async def _submit(self, operation: Callable[[threading.Event], Any]) -> Any:
cancelled = threading.Event()
future = asyncio.get_running_loop().run_in_executor(self._worker, lambda: operation(cancelled))
try:
return await asyncio.shield(future)
except asyncio.CancelledError:
cancelled.set()
# Admission stays occupied until the worker has actually stopped.
while not future.done():
try:
await asyncio.shield(future)
except asyncio.CancelledError:
continue
except Exception:
break
if not future.cancelled():
future.exception()
raise