Image-Text-to-Text
MLX
Safetensors
qwen3_5
clairic
cloudflare
systemone
structured-output
information-extraction
multimodal
custom-code
conversational
4-bit precision
Instructions to use mlx-community/clairic-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use mlx-community/clairic-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("mlx-community/clairic-4bit") config = load_config("mlx-community/clairic-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use mlx-community/clairic-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "mlx-community/clairic-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use mlx-community/clairic-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default mlx-community/clairic-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use mlx-community/clairic-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "mlx-community/clairic-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "mlx-community/clairic-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 53,474 Bytes
1a48d6b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 | """Clairic inference on Apple silicon with MLX.
Not part of the Cloudflare release: this module replaces its PyTorch engine (`native.py`, `math.py`,
`readout.py`, the typed head in `model.py` and the CUDA kernels). It keeps the same two entry points the request
pipeline calls, with the same inputs and outputs:
- `read_typed(item, media)`: one forward pass of the backbone over the typed-field prompt; the final hidden
states feed the typed head (dates, times, durations, computed numbers, estimates, forecasts);
- `async_generate(...)`: greedy decoding constrained by an xgrammar grammar (JSON schema, regex or EBNF), one
prompt per field or list item. Token log-probabilities are taken after the grammar mask, as upstream does.
The backbone is mlx-vlm's `qwen3_5` model (causal full attention every fourth layer, Gated DeltaNet linear
attention elsewhere) run through an explicit layer loop with a key/value cache for the attention layers and a
recurrent state plus convolution window for the linear layers.
"""
from __future__ import annotations
import asyncio
import concurrent.futures
import hashlib
import json
import threading
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_vlm.models.qwen3_5 import Model, ModelConfig
from mlx_vlm.models.qwen3_5.gated_delta import gated_delta_update
from . import model as cm
ATTENTION_ELEMENTS = 1 << 27 # attention scores per call (batch x heads x queries x keys); bounds temporaries
SEGMENT_TOKENS = 1024 # tokens (rows x length) run through the layers at once; longer inputs go in segments
MEMORY_FRACTION = 0.75 # default ceiling on MLX memory, as a fraction of the machine's RAM
ROPE_MODES = ("mrope", "temporal")
class PreciseRMSNorm(nn.Module):
"""RMSNorm computed in fp32 and cast back, as the reference does. The weight is stored as (1 + w) in fp32."""
def __init__(self, dims: int, eps: float) -> None:
super().__init__()
self.weight = mx.ones((dims,))
self.eps = eps
def __call__(self, x: mx.array) -> mx.array:
return mx.fast.rms_norm(x.astype(mx.float32), self.weight.astype(mx.float32), self.eps).astype(x.dtype)
@dataclass
class Cache:
"""Per-layer state after a prefix: attention keys/values, or the linear layers' memory and conv window."""
keys: list[mx.array | None]
values: list[mx.array | None]
memory: list[mx.array | None]
window: list[mx.array | None]
valid: mx.array | None = None # [B, T] False on padding; None when no row is padded
length: int = 0
position: mx.array | None = None # [B] rotary position of the next token
@classmethod
def empty(cls, layers: int) -> Cache:
return cls([None] * layers, [None] * layers, [None] * layers, [None] * layers)
def arrays(self) -> list[mx.array]:
return [a for group in (self.keys, self.values, self.memory, self.window) for a in group if a is not None]
# ---------------------------------------------------------------------------------------------- typed head
def _normalize(x: mx.array) -> mx.array:
return x / mx.maximum(mx.sqrt((x * x).sum(axis=-1, keepdims=True)), 1e-12)
def _span_scores(start: mx.array, end: mx.array, length_bias: mx.array) -> mx.array:
"""scores[q, s, l] = start[q, s] + end[q, s + l] + bias[l]; spans running past the input are -inf."""
width, tokens = length_bias.shape[0], end.shape[1]
padded = mx.concatenate([end, mx.full((end.shape[0], width - 1), -mx.inf, dtype=end.dtype)], axis=1)
index = mx.arange(tokens)[:, None] + mx.arange(width)[None, :]
return start[:, :, None] + padded[:, index] + length_bias
def _two_digits(slots: mx.array, values: range) -> mx.array:
tens = mx.array([value // 10 for value in values])
ones = mx.array([value % 10 for value in values])
return slots[:, 0, :][:, tens] + slots[:, 1, :][:, ones]
def _years(slots: mx.array) -> mx.array:
index = [[int(f"{year:04d}"[position]) for year in range(cm.YEAR_LOW, cm.YEAR_HIGH + 1)]
for position in range(cm.YEAR_DIGITS)]
inside = sum(slots[:, position, :][:, mx.array(index[position])] for position in range(cm.YEAR_DIGITS))
total = mx.logsumexp(slots, axis=-1).sum(axis=-1)
gap = mx.minimum(mx.logsumexp(inside, axis=-1) - total, -1e-06)
outside = total + mx.log(-mx.expm1(gap))
return mx.concatenate([inside, outside[:, None]], axis=-1)
class TypedHead:
"""The Clairic typed head (`clair_head.safetensors`) with the lexical readout, evaluated in fp32.
A port of upstream `ClairicHead.forward_lexical` and `answer_slot_states`: field queries are routed through
two pre-norm transformer decoder layers that cross-attend the prompt, then read out as span pointers,
digit / month / sign slots, estimate bins and list-item states.
"""
REQUIRED = {"readout": "lexical", "estimate_lexical": True, "span_trim": True, "answer_slots": True,
"answer_slot_routing": True, "number_groups": True, "number_pointer": True}
def __init__(self, path: Path, round_bf16: bool) -> None:
self.config = json.loads((path / "clair_head_config.json").read_text())
for name, value in self.REQUIRED.items():
if self.config.get(name) != value:
raise ValueError(f"unsupported typed head: {name} must be {value!r}")
if self.config.get("loop_feedback"):
raise ValueError("unsupported typed head: loop_feedback")
weights = mx.load(str(path / "clair_head.safetensors"))
if round_bf16:
# Upstream serving builds the head in bf16 before widening it to fp32 (the length bias excepted).
weights = {name: value if name == "length_bias" else value.astype(mx.bfloat16)
for name, value in weights.items()}
self.w = {name: value.astype(mx.float32) for name, value in weights.items()}
mx.eval(self.w)
self.bins = self.w["bins.weight"].shape[0]
self.pointer_width = int(self.config["pointer_width"])
self.heads = int(self.config["route_heads"])
self.layers = int(self.config["route_layers"])
self.slot_rank = int(self.config["slot_rank"])
def eval(self) -> TypedHead:
return self
# ---- building blocks
def linear(self, name: str, x: mx.array) -> mx.array:
y = x @ self.w[f"{name}.weight"].T
bias = self.w.get(f"{name}.bias")
return y if bias is None else y + bias
def norm(self, name: str, x: mx.array) -> mx.array:
return mx.fast.layer_norm(x, self.w[f"{name}.weight"], self.w[f"{name}.bias"], 1e-5)
def _project(self, prefix: str, part: int, x: mx.array) -> mx.array:
weight, bias = self.w[f"{prefix}.in_proj_weight"], self.w[f"{prefix}.in_proj_bias"]
width = weight.shape[1]
y = x @ weight[part * width:(part + 1) * width].T + bias[part * width:(part + 1) * width]
return y.reshape(*x.shape[:-1], self.heads, -1).transpose(0, 2, 1, 3)
def _attention(self, prefix: str, queries: mx.array, keys: mx.array, values: mx.array,
mask: mx.array | None) -> mx.array:
batch = queries.shape[0]
if keys.shape[0] != batch:
keys = mx.broadcast_to(keys, (batch, *keys.shape[1:]))
values = mx.broadcast_to(values, (batch, *values.shape[1:]))
out = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=queries.shape[-1] ** -0.5, mask=mask)
out = out.transpose(0, 2, 1, 3).reshape(batch, queries.shape[2], -1)
return self.linear(f"{prefix}.out_proj", out)
def _memory(self, hidden: mx.array, stop: int) -> list[tuple[mx.array, mx.array]]:
"""The prompt as cross-attention memory: each route layer's keys and values, computed once per read."""
memory = self.linear("memory_in", self.norm("token_norm", hidden[:stop]))[None]
projected = []
for layer in range(self.layers):
prefix = f"route_layers.{layer}.multihead_attn"
projected.append((self._project(prefix, 1, memory), self._project(prefix, 2, memory)))
mx.eval(projected)
return projected
def _route_layers(self, queries: mx.array, memory: list[tuple[mx.array, mx.array]],
padding: mx.array | None = None) -> mx.array:
"""`nn.TransformerDecoderLayer(norm_first=True, activation="gelu")` stack; `padding` marks ignored keys."""
mask = None if padding is None else mx.logical_not(padding)[:, None, None, :]
for layer, (keys, values) in enumerate(memory):
prefix = f"route_layers.{layer}"
x = self.norm(f"{prefix}.norm1", queries)
queries = queries + self._attention(
f"{prefix}.self_attn", self._project(f"{prefix}.self_attn", 0, x),
self._project(f"{prefix}.self_attn", 1, x), self._project(f"{prefix}.self_attn", 2, x), mask)
x = self.norm(f"{prefix}.norm2", queries)
queries = queries + self._attention(
f"{prefix}.multihead_attn", self._project(f"{prefix}.multihead_attn", 0, x), keys, values, None)
x = self.norm(f"{prefix}.norm3", queries)
queries = queries + self.linear(f"{prefix}.linear2", nn.gelu(self.linear(f"{prefix}.linear1", x)))
return queries
# ---- readouts
def typed_logits(self, raw: mx.array, routed: mx.array) -> tuple[mx.array, mx.array, mx.array]:
count, slots = raw.shape[0], cm.DIGIT_SLOTS
scale = mx.exp(mx.minimum(self.w["lexical_scale"], float(np.log(100.0))))
unit = _normalize(raw)
down = (raw @ self.w["slot_down.weight"].T).reshape(count, slots + 2, self.slot_rank)
digit_prior = (unit @ _normalize(self.w["digit_rows"]).T)[:, None, :]
month_prior = unit @ _normalize(self.w["month_rows"]).T
sign_prior = unit @ _normalize(self.w["sign_rows"]).T
residual = mx.sigmoid(self.w["residual_gate"]) * self.linear("slot_residual", routed)
digit_residual = residual[:, : slots * 10].reshape(count, slots, 10)
month_residual, sign_residual = residual[:, slots * 10: slots * 10 + 12], residual[:, slots * 10 + 12:]
return (
scale[0] * digit_prior + mx.einsum("nsr,srd->nsd", down[:, :slots], self.w["slot_up"]) + digit_residual,
scale[1] * month_prior + down[:, slots] @ self.w["month_up"] + month_residual,
scale[2] * sign_prior + down[:, slots + 1] @ self.w["sign_up"] + sign_residual,
)
def typed_outputs(self, raw: mx.array, states: mx.array, routed: mx.array) -> dict[str, mx.array]:
digits, months, signs = self.typed_logits(raw, routed)
count = raw.shape[0]
year_slots = digits[:, cm.YEAR_SLOT: cm.YEAR_SLOT + cm.YEAR_DIGITS]
return {
"number": mx.concatenate([signs, self.linear("exponent", states),
digits[:, : cm.NUMBER_DIGITS].reshape(count, -1)], axis=-1),
"date": mx.concatenate([year_slots.reshape(count, -1), _years(year_slots), months,
_two_digits(digits[:, cm.DAY_SLOT: cm.DAY_SLOT + 2], range(1, 32))], axis=-1),
"time": mx.concatenate([_two_digits(digits[:, cm.HOUR_SLOT: cm.HOUR_SLOT + 2], range(24)),
_two_digits(digits[:, cm.MINUTE_SLOT: cm.MINUTE_SLOT + 2], range(60)),
_two_digits(digits[:, cm.SECOND_SLOT: cm.SECOND_SLOT + 2], range(60))], axis=-1),
}
def group_scores(self, states: mx.array) -> mx.array:
return self.linear("number_group_up", nn.gelu(self.linear("number_group_down", states))).reshape(
-1, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)
def temper_number(self, logits: mx.array, spread: mx.array) -> mx.array:
classes, digits = cm.EXPONENT_CLASSES, cm.NUMBER_DIGITS
sign, exponent = logits[:, :2], logits[:, 2: 2 + classes]
slots = logits[:, 2 + classes:].reshape(-1, digits, 10)
limit = cm.ESTIMATE_SPREAD_LIMIT
temperature = mx.exp(limit * mx.tanh(spread / limit))
slots = mx.concatenate([slots[:, : cm.LEXICAL_DIGITS] / temperature[:, 1:, None], slots[:, cm.LEXICAL_DIGITS:]], axis=1)
return mx.concatenate([sign, exponent / temperature[:, :1], slots.reshape(slots.shape[0], -1)], axis=-1)
def forward(self, hidden: mx.array, start: int, stop: int, views: dict[str, np.ndarray],
memory: list[tuple[mx.array, mx.array]]) -> dict[str, mx.array]:
"""Upstream `forward_lexical`: `hidden` is [tokens, width] in fp32, `views` the index arrays."""
if any(name.endswith("_prefix_length") for name in views):
raise ValueError("prefixed number readouts are not supported")
rows = {name: np.asarray(value) for name, value in views.items()}
queries = rows["query_positions"]
item_kinds = [kind for kind in cm.LIST_KINDS if rows[f"{kind}_rows"].shape[0]]
positions = np.concatenate([queries] + [rows[f"{kind}_items"].reshape(-1) for kind in item_kinds])
kinds = np.zeros(positions.shape[0], dtype=np.int64)
for index, group in enumerate(cm.ROUTE_KINDS):
kinds[rows[f"{group}_rows"]] = index
offset = queries.shape[0]
for kind in item_kinds:
size = rows[f"{kind}_items"].size
kinds[offset: offset + size] = len(cm.ROUTE_KINDS) + cm.LIST_KINDS.index(kind)
offset += size
def take(values: mx.array, index: np.ndarray) -> mx.array:
return values[mx.array(index.astype(np.int32))]
raw = take(hidden, positions)
routed = self._route_layers(
(self.linear("route_in", self.norm("norm", raw)) + take(self.w["kind_embedding.weight"], kinds))[None], memory)
routed = self.norm("route_norm", routed[0])
states = self.norm("norm", raw) + mx.sigmoid(self.w["route_gate"]) * self.linear("route_out", routed)
count = queries.shape[0]
output: dict[str, mx.array] = {"null": self.linear("null", states[:count])[:, 0]}
field_states, field_raw, field_routed = states[:count], raw[:count], routed[:count]
text, tags, numbers = rows["text_rows"], rows["tags_rows"], rows["number_rows"]
scale = 1.0 / float(np.sqrt(self.pointer_width))
tokens = None
if text.shape[0] or tags.shape[0] or numbers.shape[0]:
tokens = self.norm("token_norm", hidden[start:stop])
if text.shape[0] or tags.shape[0]:
output["trim"] = self.linear("trim", tokens).reshape(-1, 2, cm.TRIM_CLASSES)
def pointer(query: str, selected: mx.array) -> mx.array:
query_start, query_end = mx.split(self.linear(query, selected), 2, axis=-1)
key_start, key_end = mx.split(self.linear("span_key", tokens), 2, axis=-1)
return _span_scores(query_start @ key_start.T * scale, query_end @ key_end.T * scale, self.w["length_bias"])
if numbers.shape[0]:
number_states = take(field_states, numbers)
output["number_copy"] = pointer("copy_query", number_states)
output["number_copy_gate"] = self.linear("copy_gate", number_states)[:, 0]
if text.shape[0]:
output["text"] = pointer("span_query", take(field_states, text))
if tags.shape[0]:
query_begin, query_inside = mx.split(self.linear("tag_query", take(field_states, tags)), 2, axis=-1)
keys = self.linear("tag_key", tokens)
begin = query_begin @ keys.T * scale + self.w["tag_bias"][0]
inside = query_inside @ keys.T * scale + self.w["tag_bias"][1]
output["tags"] = mx.stack([mx.zeros_like(begin), begin, inside], axis=-1)
typed_rows = [rows[f"{group}_rows"] for group in ("number", "date", "time")]
if any(selected.shape[0] for selected in typed_rows):
selected = np.concatenate(typed_rows)
typed = self.typed_outputs(take(field_raw, selected), take(field_states, selected), take(field_routed, selected))
offset = 0
for group, selected in zip(("number", "date", "time"), typed_rows):
if selected.shape[0]:
output[group] = typed[group][offset: offset + selected.shape[0]]
offset += selected.shape[0]
if "number" in output:
output["number_groups"] = self.group_scores(take(field_states, numbers))
estimates = rows["estimate_rows"]
if estimates.shape[0]:
estimate_states = take(field_states, estimates)
output["estimate"] = self.linear("bins", estimate_states)
output["estimate_offset"] = self.linear("offset", estimate_states)
output["estimate_overflow"] = self.linear("overflow", estimate_states)
number_logits = self.typed_outputs(take(field_raw, estimates), estimate_states, take(field_routed, estimates))["number"]
output["estimate_number_raw"] = number_logits
output["estimate_number"] = self.temper_number(number_logits, self.linear("estimate_spread", estimate_states))
output["estimate_gate"] = self.linear("estimate_gate", estimate_states)[:, 0]
offset = count
for kind in item_kinds:
shape = rows[f"{kind}_items"].shape
size = shape[0] * shape[1]
item_states, item_raw, item_routed = (
states[offset: offset + size], raw[offset: offset + size], routed[offset: offset + size])
offset += size
output[f"{kind}_end"] = self.linear("item_end", item_states)[:, 0].reshape(shape)
group = "number" if kind == "number_list" else "date"
output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[group].reshape(*shape, -1)
if kind == "number_list":
output["number_list_groups"] = self.group_scores(item_states).reshape(
*shape, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)
return output
def answer_slot_states(self, hidden: mx.array, field_queries: np.ndarray, kinds: np.ndarray, positions: np.ndarray,
lengths: np.ndarray, stop: int, memory: list[tuple[mx.array, mx.array]],
chunk: int = 256) -> mx.array:
"""Upstream `answer_slot_states`: each answer-slot position corrected by evidence routed from the prompt."""
width = positions.shape[1]
if (width < 1 or positions.shape[0] != field_queries.shape[0] or (lengths < 1).any() or (lengths > width).any()
or not 0 < stop <= hidden.shape[0]):
raise ValueError("invalid answer-slot routing layout")
gate = mx.sigmoid(self.w["route_gate"])
output = []
field_batch = max(1, min(chunk // (width + 1), 262144 // stop))
for first in range(0, positions.shape[0], field_batch):
last = first + field_batch
slot_positions = mx.array(positions[first:last].astype(np.int32))
query_positions = mx.concatenate([mx.array(field_queries[first:last, None].astype(np.int32)), slot_positions], axis=1)
row_lengths = mx.array(lengths[first:last].astype(np.int32))
row_kinds = mx.array(kinds[first:last].astype(np.int32))
padding = mx.arange(width + 1)[None, :] > row_lengths[:, None]
queries = self.linear("route_in", self.norm("norm", hidden[query_positions]))
queries = self._route_layers(queries + self.w["kind_embedding.weight"][row_kinds][:, None, :], memory, padding)
evidence = self.linear("route_out", self.norm("route_norm", queries[:, 1:]))
scale = mx.tanh(self.w["slot_route_scale"][row_kinds])[:, None, None]
corrected = hidden[slot_positions] + scale * gate * evidence
index = mx.minimum(mx.arange(width)[None, :], row_lengths[:, None] - 1)
output.append(mx.take_along_axis(corrected, index[:, :, None], axis=1))
mx.eval(output[-1])
return mx.concatenate(output, axis=0)
# ---------------------------------------------------------------------------------------------- engine
@dataclass
class _Rows:
"""One right-padded batch: token ids, which positions are real, and the rows' image inputs in order."""
ids: np.ndarray
valid: np.ndarray
media: list[dict[str, Any]] = field(default_factory=list)
@property
def lengths(self) -> np.ndarray:
return self.valid.sum(axis=1)
def _common_prefix(sequences: list[list[int]]) -> int:
first, last = min(sequences), max(sequences)
return next((i for i, (a, b) in enumerate(zip(first, last)) if a != b), min(len(first), len(last)))
class MLXEngine:
"""Share one MLX backbone between grammar-constrained extraction and typed readouts."""
def __init__(self, path: Path, tokenizer: Any, *, batch_size: int = 8, batch_token_budget: int = 8192,
rope: str = "temporal", device: str = "gpu", memory_limit_gb: float | None = None) -> None:
import xgrammar as xgr
if rope not in ROPE_MODES:
raise ValueError(f"rope must be one of {ROPE_MODES}")
self.path, self.tokenizer, self.rope = Path(path), tokenizer, rope
self.batch_size, self.batch_token_budget = max(1, int(batch_size)), max(1, int(batch_token_budget))
self.text_batch_size = self.batch_size
self.device = device # "gpu", or "cpu" for bit-accurate float32 checks (slow)
self.capture: Callable[[str, Any], None] | None = None # test hook: receives intermediate arrays
# A request that would need more memory than this fails with an error instead of exhausting the machine.
self.memory_cap = int((memory_limit_gb * 1e9) if memory_limit_gb else MEMORY_FRACTION * mx.device_info()["memory_size"])
self._worker = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="clairic-model")
self._features: dict[str, mx.array] = {}
self._prefixes: dict[tuple, Cache] = {}
self.prefix_cache = True # read a request's images once and continue every prompt from that state
# Every MLX call happens on the worker thread, including loading the weights.
self._worker.submit(self._load).result()
self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
self.pad = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else self.eos
self.vocab = self.lm.lm_head.weight.shape[0]
info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=self.vocab, stop_token_ids=[self.eos])
self.compiler = xgr.GrammarCompiler(info, max_threads=2)
self._xgr = xgr
# ---- loading
def _load(self) -> None:
path = self.path
if self.device == "cpu":
mx.set_default_device(mx.cpu)
config = json.loads((path / "config.json").read_text())
self.model = Model(ModelConfig.from_dict(config))
self.lm = self.model.language_model
text = self.lm.model
hidden, eps = config["text_config"]["hidden_size"], config["text_config"]["rms_norm_eps"]
text.norm = PreciseRMSNorm(hidden, eps)
for layer in text.layers:
layer.input_layernorm = PreciseRMSNorm(hidden, eps)
layer.post_attention_layernorm = PreciseRMSNorm(hidden, eps)
if not layer.is_linear:
layer.self_attn.q_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
layer.self_attn.k_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
weights: dict[str, mx.array] = {}
for shard in sorted(path.glob("model*.safetensors")):
weights.update(mx.load(str(shard)))
if quantization := config.get("quantization"):
def quantized(path: str, module: nn.Module) -> bool | dict:
if not hasattr(module, "to_quantized") or f"{path}.scales" not in weights:
return False
settings = quantization.get(path) # modules with their own bit width
return settings if isinstance(settings, dict) else True
nn.quantize(self.model, group_size=quantization["group_size"], bits=quantization["bits"],
mode=quantization.get("mode", "affine"), class_predicate=quantized)
self.model.load_weights(list(weights.items()), strict=True)
self.model.eval()
mx.eval(self.model.parameters())
self.image_token = int(config["image_token_id"])
self.vision_end = int(config["vision_end_token_id"])
self.dtype = text.layers[0].linear_attn.conv1d.weight.dtype # never quantized: bf16 (release) or fp32
self.backbone = self.model
self.head = TypedHead(path, round_bf16=self.dtype != mx.float32)
# ---- request scope (image features are kept for one request)
def begin_request(self) -> None:
self._features, self._prefixes = {}, {}
async def end_request(self) -> None:
await asyncio.get_running_loop().run_in_executor(self._worker, self._clear)
def _clear(self) -> None:
self._features, self._prefixes = {}, {}
mx.clear_cache()
def shutdown(self) -> None:
self._worker.shutdown(wait=True, cancel_futures=True)
self._features, self._prefixes = {}, {}
def _check_memory(self) -> None:
if mx.get_active_memory() > self.memory_cap:
mx.clear_cache()
if mx.get_active_memory() > self.memory_cap:
raise MemoryError(
f"this request needs more than the {self.memory_cap / 1e9:.0f} GB memory limit; "
"shorten the input, lower batch_token_budget, or raise memory_limit_gb")
# ---- backbone
def _image_features(self, mapping: dict[str, Any]) -> mx.array:
key = mapping.get("clair_pixels_key")
if key is None:
pixels = np.asarray(mapping["pixel_values"])
key = hashlib.sha1(pixels.tobytes() + np.asarray(mapping["image_grid_thw"]).tobytes()).hexdigest()
if key not in self._features:
tower = self.model.vision_tower
pixels = mx.array(mapping["pixel_values"]).astype(tower.patch_embed.proj.weight.dtype)
features, _ = tower(pixels, mx.array(np.asarray(mapping["image_grid_thw"])))
mx.eval(features)
self._features[key] = features
return self._features[key]
def _rows(self, sequences: list[list[int]], media: list[Any]) -> _Rows:
width = max(map(len, sequences))
ids = np.full((len(sequences), width), self.pad, dtype=np.int32)
valid = np.zeros((len(sequences), width), dtype=bool)
for row, sequence in enumerate(sequences):
ids[row, : len(sequence)] = sequence
valid[row, : len(sequence)] = True
return _Rows(ids, valid, [mapping for payload in media if payload for mapping in payload])
def _embed(self, rows: _Rows) -> tuple[mx.array, mx.array]:
"""Input embeddings (image features merged in) and rotary positions [3, B, L] of a right-padded batch."""
tokens, mask = mx.array(rows.ids), mx.array(rows.valid.astype(np.int32))
if rows.media:
features = mx.concatenate([self._image_features(mapping) for mapping in rows.media], axis=0)
grids = mx.array(np.concatenate([np.asarray(mapping["image_grid_thw"]) for mapping in rows.media]))
embedded = self.model.get_input_embeddings(
tokens, features, image_grid_thw=grids, mask=mask, cached_image_features=features)
else:
embedded = self.model.get_input_embeddings(tokens, None, mask=mask)
positions = embedded.position_ids
if positions.ndim == 2: # text only: the same position on the three rotary axes
positions = mx.broadcast_to(positions[None], (3, *positions.shape))
return embedded.inputs_embeds, positions
def _rotary(self, positions: mx.array) -> mx.array:
# "temporal" (default) is what the upstream runtime does in bfloat16, its serving configuration: the
# rotary table is indexed by the first (temporal) position axis for every frequency, so the tokens of an
# image share one rotary position. "mrope" is the stock Transformers 3-axis rotary, which upstream runs
# in float32. Text tokens are identical in both.
if self.rope == "temporal":
return mx.broadcast_to(positions[:1], positions.shape)
return positions
def _linear(self, attention: nn.Module, x: mx.array, valid: mx.array | None, window: mx.array | None,
memory: mx.array | None, lengths: mx.array | None = None) -> tuple[mx.array, mx.array, mx.array]:
"""A Gated DeltaNet layer continuing from a convolution window and recurrent memory (padding skipped)."""
batch, length, _ = x.shape
mixed = attention.in_proj_qkv(x)
z = attention.in_proj_z(x).reshape(batch, length, -1, attention.head_v_dim)
b, a = attention.in_proj_b(x), attention.in_proj_a(x)
if valid is not None:
mixed = mx.where(valid[..., None], mixed, 0)
if window is None:
window = mx.zeros((batch, attention.conv_kernel_size - 1, attention.conv_dim), dtype=mixed.dtype)
conv_input = mx.concatenate([window.astype(mixed.dtype), mixed], axis=1)
conv = nn.silu(attention.conv1d(conv_input))
q, k, v = (
t.reshape(batch, length, heads, width)
for t, heads, width in zip(
mx.split(conv, [attention.key_dim, 2 * attention.key_dim], -1),
[attention.num_k_heads, attention.num_k_heads, attention.num_v_heads],
[attention.head_k_dim, attention.head_k_dim, attention.head_v_dim], strict=True)
)
q, k = attention._normalize_qk(q, k)
out, memory = gated_delta_update(q, k, v, a, b, attention.A_log, attention.dt_bias, state=memory, mask=valid,
use_kernel=True)
out = attention.out_proj(attention.norm(out, z).reshape(batch, length, -1))
taps = attention.conv_kernel_size - 1
if lengths is None:
return out, memory, conv_input[:, -taps:]
# Right-padded rows: keep the last real inputs of each row, not the padding after them.
index = lengths[:, None] + mx.arange(taps)[None, :]
index = mx.broadcast_to(index[:, :, None], (batch, taps, conv_input.shape[-1]))
return out, memory, mx.take_along_axis(conv_input, index, axis=1)
def _attention(self, attention: nn.Module, x: mx.array, positions: mx.array, keys: mx.array | None,
values: mx.array | None, valid: mx.array | None, offset: int) -> tuple[mx.array, mx.array, mx.array]:
"""Causal attention of new tokens over the cached and new keys; padded keys are masked."""
batch, length, _ = x.shape
queries, new_keys, new_values, gate, _ = attention._prepare_projected_qkv(
attention.q_proj(x), attention.k_proj(x), attention.v_proj(x), None, positions, None, None)
if keys is not None:
new_keys = mx.concatenate([keys, new_keys], axis=2)
new_values = mx.concatenate([values, new_values], axis=2)
total = new_keys.shape[2]
# The score matrix is queries x keys per head. Long inputs are attended in slices of query rows so its
# size stays bounded (a slice's output depends only on its own rows); unsliced, a 40k-token input needs
# tens of GB per layer.
rows = max(1, ATTENTION_ELEMENTS // (batch * attention.num_attention_heads * total))
outputs = []
for start in range(0, length, rows):
stop = min(length, start + rows)
mask = None
if length > 1:
key_index = mx.arange(total)[None, :]
row_index = mx.arange(offset + start, offset + stop)[:, None]
mask = (key_index <= row_index)[None, None]
if valid is not None: # a padded query still sees itself, so no row is fully masked
mask = mask & (valid[:, None, None, :] | (key_index == row_index)[None, None])
elif valid is not None:
mask = valid[:, None, None, :]
outputs.append(mx.fast.scaled_dot_product_attention(
queries[:, :, start:stop], new_keys, new_values, scale=attention.scale, mask=mask))
if rows < length:
mx.eval(outputs[-1])
self._check_memory()
output = outputs[0] if len(outputs) == 1 else mx.concatenate(outputs, axis=2)
output = output.transpose(0, 2, 1, 3).reshape(batch, length, -1)
return attention.o_proj(output * mx.sigmoid(gate)), new_keys, new_values
def _forward(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
keep: bool = True) -> tuple[mx.array, Cache | None]:
"""Run the layers over new tokens, optionally continuing from a cache.
`lengths` gives each row's number of real tokens when the batch is right-padded (None: no padding).
Returns the hidden states before the final norm and, with `keep`, the cache after each row's last real
token (padded positions stay in the attention cache, marked invalid).
"""
layers = self.lm.model.layers
cache = cache or Cache.empty(len(layers))
batch, length, _ = h.shape
valid = row_lengths = None
if lengths is not None and (lengths < length).any():
row_lengths = mx.array(lengths.astype(np.int32))
valid = mx.arange(length)[None, :] < row_lengths[:, None]
all_valid = None
if cache.valid is not None or valid is not None:
before = cache.valid if cache.valid is not None else mx.ones((batch, cache.length), dtype=mx.bool_)
new = valid if valid is not None else mx.ones((batch, length), dtype=mx.bool_)
all_valid = mx.concatenate([before, new], axis=1)
rotary = self._rotary(positions)
updated = Cache.empty(len(layers)) if keep else None
stepwise = batch * length > 4096 # keep the lazy graph small on long inputs
for index, layer in enumerate(layers):
x = layer.input_layernorm(h)
if layer.is_linear:
r, memory, window = self._linear(layer.linear_attn, x, valid, cache.window[index], cache.memory[index],
row_lengths)
if keep:
updated.memory[index], updated.window[index] = memory, window
else:
r, keys, values = self._attention(layer.self_attn, x, rotary, cache.keys[index], cache.values[index],
all_valid, cache.length)
if keep:
updated.keys[index], updated.values[index] = keys, values
h = h + r
h = h + layer.mlp(layer.post_attention_layernorm(h))
if stepwise:
mx.eval(h, *(updated.arrays() if keep else []))
self._check_memory()
if keep:
updated.valid, updated.length = all_valid, cache.length + length
return h, updated
def _run(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
keep: bool = True) -> tuple[mx.array, Cache | None]:
"""`_forward` over a whole input. Long inputs go through the layers in segments, each continuing from
the cache of the previous one, so working memory depends on the segment size rather than the input
length (attention is causal and the linear layers are recurrent, so the result is the same). With
`keep`, the returned cache has `position` set to each row's next rotary position."""
batch, length, _ = h.shape
segment = max(256, SEGMENT_TOKENS // batch)
if length <= segment:
out, cache = self._forward(h, positions, lengths, cache, keep)
else:
outputs = []
for start in range(0, length, segment):
stop = min(length, start + segment)
inside = None if lengths is None else np.clip(lengths - start, 0, stop - start)
part, cache = self._forward(h[:, start:stop], positions[:, :, start:stop], inside, cache)
mx.eval(part, *cache.arrays())
self._check_memory()
outputs.append(part)
out = mx.concatenate(outputs, axis=1)
cache = cache if keep else None
if keep:
last = mx.array((lengths if lengths is not None else np.full(batch, length)).astype(np.int32)) - 1
index = mx.broadcast_to(last[None, :, None], (positions.shape[0], batch, 1))
cache.position = mx.take_along_axis(positions, index, axis=2)[:, :, 0].max(axis=0) + 1
return out, cache
def _hidden(self, sequences: list[list[int]], media: list[Any]) -> mx.array:
"""Final-norm hidden states of an unpadded single sequence."""
rows = self._rows(sequences, media)
h, positions = self._embed(rows)
h, _ = self._run(h, positions, None, None, keep=False)
return self.lm.model.norm(h)
# ---- typed readout (upstream `read_record`)
def _read_record(self, states: mx.array, item: Any) -> dict[str, np.ndarray]:
start, length, kinds, queries, items, slot_fields, slots = cm.parse_prediction_layout(
cm.prediction_layout(item), len(item.input_ids), with_slots=True)
views = cm.prediction_views(kinds, queries, items, None)
hidden = states.astype(mx.float32)
stop = start + length
memory = self.head._memory(hidden, stop)
output = self.head.forward(hidden, start, stop, views, memory)
mx.eval(output)
values: dict[str, np.ndarray] = {name: np.array(value) for name, value in output.items()}
if slots:
width = max(map(len, slots))
positions = np.array([row + [row[-1]] * (width - len(row)) for row in slots], dtype=np.int64)
field_queries = np.asarray(views["query_positions"])[slot_fields]
slot_kinds = np.array([cm.ROUTE_KINDS.index("tags" if kinds[row] == "text_list" else kinds[row])
for row in slot_fields], dtype=np.int64)
lengths = np.array([len(row) for row in slots], dtype=np.int64)
slot_states = self.head.answer_slot_states(hidden, field_queries, slot_kinds, positions, lengths, stop, memory)
slot_states = slot_states.reshape(-1, slot_states.shape[-1]).astype(states.dtype)
ids, logprobs = [], []
for first in range(0, slot_states.shape[0], 256):
logits = self.lm.lm_head(slot_states[first: first + 256]).astype(mx.float32)
best = mx.argmax(logits, axis=-1)
ids.append(best)
logprobs.append(mx.take_along_axis(logits, best[:, None], axis=-1)[:, 0] - mx.logsumexp(logits, axis=-1))
mx.eval(ids[-1], logprobs[-1])
values["slot_ids"] = np.array(mx.concatenate(ids)).reshape(positions.shape)
values["slot_log_probs"] = np.array(mx.concatenate(logprobs)).reshape(positions.shape)
return values
def _read_typed(self, item: Any, media: Any, _cancelled: threading.Event | None = None) -> dict[str, dict[str, Any]]:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
states = self._hidden([list(item.input_ids)], [media])[0]
mx.eval(states)
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
values = self._read_record(states, item)
compact = cm.compact_output(values)
if self.capture is not None:
self.capture("hidden", np.array(states.astype(mx.float32)))
self.capture("compact", np.asarray(compact, dtype=np.float32))
if "slot_ids" in values:
self.capture("slots", values["slot_ids"])
if media:
self.capture("pixels", np.asarray(media[0]["pixel_values"]).astype(np.float16))
self.capture("grid", np.asarray(media[0]["image_grid_thw"]))
values = cm.expand_output(compact, item, bins=self.head.bins, estimate_lexical=True, span_trim=True,
number_groups=True, number_pointer=True)
return cm.decode_record(values, item, self.tokenizer)
async def read_typed(self, item: Any, media: Any = None) -> dict[str, dict[str, Any]]:
return await self._submit(lambda cancelled: self._read_typed(item, media, _cancelled=cancelled))
# ---- constrained generation
def _grammar(self, parameters: dict[str, Any]) -> Any:
if "json_schema" in parameters:
value = parameters["json_schema"]
return self.compiler.compile_json_schema(json.loads(value) if isinstance(value, str) else value,
any_whitespace=True)
if "regex" in parameters:
return self.compiler.compile_regex(parameters["regex"])
if "ebnf" in parameters:
return self.compiler.compile_grammar(parameters["ebnf"])
raise ValueError("Extraction continuations require a JSON schema, regex, or grammar")
def _logits(self, h: mx.array) -> np.ndarray:
"""Vocabulary logits (fp32) from hidden states [B, 1, width] before the final norm."""
return np.array(self.lm.lm_head(self.lm.model.norm(h))[:, 0].astype(mx.float32))
@staticmethod
def _media_key(payload: Any) -> tuple | None:
return tuple(mapping.get("clair_pixels_key") or id(mapping["pixel_values"]) for mapping in payload) if payload else None
def _image_prefix(self, sequences: list[list[int]], media: list[Any]) -> tuple[Cache | None, int]:
"""The state after the images shared by every prompt of a batch, computed once per request.
Image prompts start with the same image tokens and differ only afterwards. The shared part, up to the
last `<|vision_end|>` common to all rows, is run once and each row continues from its cache; attention
is causal and the linear layers are recurrent, so this equals running every full prompt.
"""
if not self.prefix_cache or not media[0] or any(self._media_key(m) != self._media_key(media[0]) for m in media):
return None, 0
common = sequences[0][: _common_prefix(sequences)]
if self.vision_end not in common:
return None, 0
cut = len(common) - common[::-1].index(self.vision_end)
if any(self.image_token in sequence[cut:] for sequence in sequences):
return None, 0
key = (self._media_key(media[0]), hashlib.sha1(np.asarray(common[:cut], dtype=np.int32).tobytes()).hexdigest())
if key not in self._prefixes:
rows = self._rows([common[:cut]], [media[0]])
h, positions = self._embed(rows)
_, cache = self._run(h, positions, None, None)
mx.eval(cache.position, *cache.arrays())
self._prefixes[key] = cache
return self._prefixes[key], cut
def _prefill(self, sequences: list[list[int]], media: list[Any]) -> tuple[mx.array, Cache]:
"""Run the prompts of one batch. Returns each row's last hidden state [B, 1, width] and the cache."""
size = len(sequences)
prefix, cut = self._image_prefix(sequences, media)
if prefix is None:
rows = self._rows(sequences, media)
h, positions = self._embed(rows)
cache = None
else:
rows = self._rows([sequence[cut:] for sequence in sequences], [None] * size)
h = self.lm.model.embed_tokens(mx.array(rows.ids))
positions = prefix.position[0] + mx.arange(rows.ids.shape[1])
positions = mx.broadcast_to(positions[None, None, :], (3, size, rows.ids.shape[1]))
def spread(array: mx.array | None) -> mx.array | None:
return None if array is None else mx.broadcast_to(array, (size, *array.shape[1:]))
cache = Cache([spread(a) for a in prefix.keys], [spread(a) for a in prefix.values],
[spread(a) for a in prefix.memory], [spread(a) for a in prefix.window], None, prefix.length)
lengths = rows.lengths
h, cache = self._run(h, positions, lengths, cache)
last = mx.array((lengths - 1).astype(np.int32))
index = mx.broadcast_to(last[:, None, None], (size, 1, h.shape[-1]))
return mx.take_along_axis(h, index, axis=1), cache
def _decode(self, sequences: list[list[int]], media: list[Any], grammars: list[Any], maximum: int,
stop_ids: tuple[int, ...], score_ids: list[int], cancelled: threading.Event | None,
) -> list[tuple[list[int], list[float], list[float]]]:
"""Greedy decoding of one batch under per-row grammars. Returns (tokens, log-probabilities, first-step
log-probabilities of `score_ids`) per row; tokens end with the stop token when one was produced."""
size = len(sequences)
h, cache = self._prefill(sequences, media)
mx.eval(h, cache.position, *cache.arrays())
scores = self._logits(h)
matchers = [self._xgr.GrammarMatcher(grammar) for grammar in grammars]
bitmask = np.zeros(self._xgr.get_bitmask_shape(size, self.vocab), dtype=np.int32)
tokens: list[list[int]] = [[] for _ in range(size)]
logprobs: list[list[float]] = [[] for _ in range(size)]
first: list[list[float]] = [[] for _ in range(size)]
done = [False] * size
for step in range(maximum):
for row, matcher in enumerate(matchers):
if not matcher.is_terminated():
matcher.fill_next_token_bitmask(bitmask, row)
allowed = np.unpackbits(bitmask.view(np.uint8), axis=-1, bitorder="little")[:, : self.vocab].astype(bool)
masked = np.where(allowed, scores, -np.inf).astype(np.float32)
peak = masked.max(axis=-1, keepdims=True)
with np.errstate(divide="ignore"):
log_probabilities = masked - peak - np.log(np.exp(masked - peak).sum(axis=-1, keepdims=True, dtype=np.float32))
choices = masked.argmax(axis=-1)
for row in range(size):
if step == 0:
first[row] = [float(log_probabilities[row, token]) for token in score_ids]
if done[row]:
continue
token = int(choices[row])
tokens[row].append(token)
logprobs[row].append(float(log_probabilities[row, token]))
if token in stop_ids:
done[row] = True
if not matchers[row].is_terminated() and not matchers[row].accept_token(token):
raise RuntimeError("Generated token rejected by its grammar")
if all(done) or step + 1 == maximum:
break
if cancelled is not None and cancelled.is_set():
raise RuntimeError("Inference cancelled")
feed = mx.array([[self.pad if done[row] else tokens[row][-1]] for row in range(size)])
position = cache.position
h, cache = self._forward(self.lm.model.embed_tokens(feed), mx.broadcast_to(position[None, :, None], (3, size, 1)),
None, cache)
cache.position = position + 1
mx.eval(h, cache.position, *cache.arrays())
scores = self._logits(h)
return list(zip(tokens, logprobs, first, strict=True))
def _generate(self, *, input_ids: list[int] | list[list[int]], image_data: Any = None,
sampling_params: dict[str, Any] | list[dict[str, Any]], rid: Any = None,
token_ids_logprob: list[int] | None = None, _cancelled: threading.Event | None = None,
**kwargs: Any) -> Any:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
if not input_ids:
return []
single = isinstance(input_ids[0], int)
sequences = [input_ids] if single else input_ids
size = len(sequences)
parameters = [sampling_params] * size if isinstance(sampling_params, dict) else sampling_params
media = [image_data] if single else (image_data or [None] * size)
ids = [rid] if single else (rid or [None] * size)
if not (len(parameters) == len(media) == len(ids) == size):
raise ValueError("Extraction batch arguments have inconsistent lengths")
results: list[Any] = [None] * size
groups: dict[tuple[int, tuple[int, ...]], list[int]] = defaultdict(list)
for index, params in enumerate(parameters):
if params.get("temperature", 0) != 0:
raise ValueError("Clairic extraction uses deterministic decoding")
groups[(int(params.get("max_new_tokens", 256)), tuple(params.get("stop_token_ids", [self.eos])))].append(index)
for (maximum, stop_ids), indices in groups.items():
# Rows with the same images and similar length share a batch; a row's result does not depend on
# its batch. Rows that share images are charged for the image prefix once.
indices = sorted(indices, key=lambda index: (str(self._media_key(media[index])), len(sequences[index])))
chunks: list[list[int]] = []
pending: list[int] = []
for index in indices:
if pending and (len(pending) >= self.batch_size
or self._media_key(media[index]) != self._media_key(media[pending[0]])
or self._cost([sequences[i] for i in (*pending, index)], media[index]) > self.batch_token_budget):
chunks.append(pending)
pending = []
pending.append(index)
if pending:
chunks.append(pending)
for chunk in chunks:
if _cancelled is not None and _cancelled.is_set():
raise RuntimeError("Inference cancelled")
decoded = self._decode([sequences[i] for i in chunk], [media[i] for i in chunk],
[self._grammar(parameters[i]) for i in chunk], maximum, stop_ids,
list(token_ids_logprob or []), _cancelled)
for index, (output, logprobs, first) in zip(chunk, decoded, strict=True):
stopped = bool(output) and output[-1] in stop_ids
results[index] = {
"output_ids": output,
"text": self.tokenizer.decode(output[:-1] if stopped else output, skip_special_tokens=False,
clean_up_tokenization_spaces=False),
"meta_info": {
"id": ids[index],
"completion_tokens": len(output),
"finish_reason": {"type": "stop", "matched": output[-1]} if stopped else {"type": "length"},
"output_token_logprobs": [(value, token, None) for value, token in zip(logprobs, output, strict=True)],
"output_token_ids_logprobs": [[(value, token, None) for value, token in
zip(first, token_ids_logprob or [], strict=True)]],
},
}
return results[0] if single else results
def _cost(self, sequences: list[list[int]], payload: Any) -> int:
"""Tokens a batch runs through the model: the padded width times its rows (shared image tokens once)."""
shared = 0
if payload and self.prefix_cache and len(sequences) > 1:
common = sequences[0][: _common_prefix(sequences)]
shared = len(common) - common[::-1].index(self.vision_end) if self.vision_end in common else 0
return (max(map(len, sequences)) - shared) * len(sequences)
async def async_generate(self, **kwargs: Any) -> Any:
return await self._submit(lambda cancelled: self._generate(_cancelled=cancelled, **kwargs))
async def _submit(self, operation: Callable[[threading.Event], Any]) -> Any:
cancelled = threading.Event()
future = asyncio.get_running_loop().run_in_executor(self._worker, lambda: operation(cancelled))
try:
return await asyncio.shield(future)
except asyncio.CancelledError:
cancelled.set()
# Admission stays occupied until the worker has actually stopped.
while not future.done():
try:
await asyncio.shield(future)
except asyncio.CancelledError:
continue
except Exception:
break
if not future.cancelled():
future.exception()
raise
|