File size: 53,474 Bytes
1a48d6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
"""Clairic inference on Apple silicon with MLX.

Not part of the Cloudflare release: this module replaces its PyTorch engine (`native.py`, `math.py`,
`readout.py`, the typed head in `model.py` and the CUDA kernels). It keeps the same two entry points the request
pipeline calls, with the same inputs and outputs:

- `read_typed(item, media)`: one forward pass of the backbone over the typed-field prompt; the final hidden
  states feed the typed head (dates, times, durations, computed numbers, estimates, forecasts);
- `async_generate(...)`: greedy decoding constrained by an xgrammar grammar (JSON schema, regex or EBNF), one
  prompt per field or list item. Token log-probabilities are taken after the grammar mask, as upstream does.

The backbone is mlx-vlm's `qwen3_5` model (causal full attention every fourth layer, Gated DeltaNet linear
attention elsewhere) run through an explicit layer loop with a key/value cache for the attention layers and a
recurrent state plus convolution window for the linear layers.
"""

from __future__ import annotations

import asyncio
import concurrent.futures
import hashlib
import json
import threading
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx_vlm.models.qwen3_5 import Model, ModelConfig
from mlx_vlm.models.qwen3_5.gated_delta import gated_delta_update

from . import model as cm

ATTENTION_ELEMENTS = 1 << 27  # attention scores per call (batch x heads x queries x keys); bounds temporaries
SEGMENT_TOKENS = 1024  # tokens (rows x length) run through the layers at once; longer inputs go in segments
MEMORY_FRACTION = 0.75  # default ceiling on MLX memory, as a fraction of the machine's RAM
ROPE_MODES = ("mrope", "temporal")


class PreciseRMSNorm(nn.Module):
    """RMSNorm computed in fp32 and cast back, as the reference does. The weight is stored as (1 + w) in fp32."""

    def __init__(self, dims: int, eps: float) -> None:
        super().__init__()
        self.weight = mx.ones((dims,))
        self.eps = eps

    def __call__(self, x: mx.array) -> mx.array:
        return mx.fast.rms_norm(x.astype(mx.float32), self.weight.astype(mx.float32), self.eps).astype(x.dtype)


@dataclass
class Cache:
    """Per-layer state after a prefix: attention keys/values, or the linear layers' memory and conv window."""

    keys: list[mx.array | None]
    values: list[mx.array | None]
    memory: list[mx.array | None]
    window: list[mx.array | None]
    valid: mx.array | None = None  # [B, T] False on padding; None when no row is padded
    length: int = 0
    position: mx.array | None = None  # [B] rotary position of the next token

    @classmethod
    def empty(cls, layers: int) -> Cache:
        return cls([None] * layers, [None] * layers, [None] * layers, [None] * layers)

    def arrays(self) -> list[mx.array]:
        return [a for group in (self.keys, self.values, self.memory, self.window) for a in group if a is not None]


# ---------------------------------------------------------------------------------------------- typed head
def _normalize(x: mx.array) -> mx.array:
    return x / mx.maximum(mx.sqrt((x * x).sum(axis=-1, keepdims=True)), 1e-12)


def _span_scores(start: mx.array, end: mx.array, length_bias: mx.array) -> mx.array:
    """scores[q, s, l] = start[q, s] + end[q, s + l] + bias[l]; spans running past the input are -inf."""
    width, tokens = length_bias.shape[0], end.shape[1]
    padded = mx.concatenate([end, mx.full((end.shape[0], width - 1), -mx.inf, dtype=end.dtype)], axis=1)
    index = mx.arange(tokens)[:, None] + mx.arange(width)[None, :]
    return start[:, :, None] + padded[:, index] + length_bias


def _two_digits(slots: mx.array, values: range) -> mx.array:
    tens = mx.array([value // 10 for value in values])
    ones = mx.array([value % 10 for value in values])
    return slots[:, 0, :][:, tens] + slots[:, 1, :][:, ones]


def _years(slots: mx.array) -> mx.array:
    index = [[int(f"{year:04d}"[position]) for year in range(cm.YEAR_LOW, cm.YEAR_HIGH + 1)]
             for position in range(cm.YEAR_DIGITS)]
    inside = sum(slots[:, position, :][:, mx.array(index[position])] for position in range(cm.YEAR_DIGITS))
    total = mx.logsumexp(slots, axis=-1).sum(axis=-1)
    gap = mx.minimum(mx.logsumexp(inside, axis=-1) - total, -1e-06)
    outside = total + mx.log(-mx.expm1(gap))
    return mx.concatenate([inside, outside[:, None]], axis=-1)


class TypedHead:
    """The Clairic typed head (`clair_head.safetensors`) with the lexical readout, evaluated in fp32.

    A port of upstream `ClairicHead.forward_lexical` and `answer_slot_states`: field queries are routed through
    two pre-norm transformer decoder layers that cross-attend the prompt, then read out as span pointers,
    digit / month / sign slots, estimate bins and list-item states.
    """

    REQUIRED = {"readout": "lexical", "estimate_lexical": True, "span_trim": True, "answer_slots": True,
                "answer_slot_routing": True, "number_groups": True, "number_pointer": True}

    def __init__(self, path: Path, round_bf16: bool) -> None:
        self.config = json.loads((path / "clair_head_config.json").read_text())
        for name, value in self.REQUIRED.items():
            if self.config.get(name) != value:
                raise ValueError(f"unsupported typed head: {name} must be {value!r}")
        if self.config.get("loop_feedback"):
            raise ValueError("unsupported typed head: loop_feedback")
        weights = mx.load(str(path / "clair_head.safetensors"))
        if round_bf16:
            # Upstream serving builds the head in bf16 before widening it to fp32 (the length bias excepted).
            weights = {name: value if name == "length_bias" else value.astype(mx.bfloat16)
                       for name, value in weights.items()}
        self.w = {name: value.astype(mx.float32) for name, value in weights.items()}
        mx.eval(self.w)
        self.bins = self.w["bins.weight"].shape[0]
        self.pointer_width = int(self.config["pointer_width"])
        self.heads = int(self.config["route_heads"])
        self.layers = int(self.config["route_layers"])
        self.slot_rank = int(self.config["slot_rank"])

    def eval(self) -> TypedHead:
        return self

    # ---- building blocks
    def linear(self, name: str, x: mx.array) -> mx.array:
        y = x @ self.w[f"{name}.weight"].T
        bias = self.w.get(f"{name}.bias")
        return y if bias is None else y + bias

    def norm(self, name: str, x: mx.array) -> mx.array:
        return mx.fast.layer_norm(x, self.w[f"{name}.weight"], self.w[f"{name}.bias"], 1e-5)

    def _project(self, prefix: str, part: int, x: mx.array) -> mx.array:
        weight, bias = self.w[f"{prefix}.in_proj_weight"], self.w[f"{prefix}.in_proj_bias"]
        width = weight.shape[1]
        y = x @ weight[part * width:(part + 1) * width].T + bias[part * width:(part + 1) * width]
        return y.reshape(*x.shape[:-1], self.heads, -1).transpose(0, 2, 1, 3)

    def _attention(self, prefix: str, queries: mx.array, keys: mx.array, values: mx.array,
                   mask: mx.array | None) -> mx.array:
        batch = queries.shape[0]
        if keys.shape[0] != batch:
            keys = mx.broadcast_to(keys, (batch, *keys.shape[1:]))
            values = mx.broadcast_to(values, (batch, *values.shape[1:]))
        out = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=queries.shape[-1] ** -0.5, mask=mask)
        out = out.transpose(0, 2, 1, 3).reshape(batch, queries.shape[2], -1)
        return self.linear(f"{prefix}.out_proj", out)

    def _memory(self, hidden: mx.array, stop: int) -> list[tuple[mx.array, mx.array]]:
        """The prompt as cross-attention memory: each route layer's keys and values, computed once per read."""
        memory = self.linear("memory_in", self.norm("token_norm", hidden[:stop]))[None]
        projected = []
        for layer in range(self.layers):
            prefix = f"route_layers.{layer}.multihead_attn"
            projected.append((self._project(prefix, 1, memory), self._project(prefix, 2, memory)))
        mx.eval(projected)
        return projected

    def _route_layers(self, queries: mx.array, memory: list[tuple[mx.array, mx.array]],
                      padding: mx.array | None = None) -> mx.array:
        """`nn.TransformerDecoderLayer(norm_first=True, activation="gelu")` stack; `padding` marks ignored keys."""
        mask = None if padding is None else mx.logical_not(padding)[:, None, None, :]
        for layer, (keys, values) in enumerate(memory):
            prefix = f"route_layers.{layer}"
            x = self.norm(f"{prefix}.norm1", queries)
            queries = queries + self._attention(
                f"{prefix}.self_attn", self._project(f"{prefix}.self_attn", 0, x),
                self._project(f"{prefix}.self_attn", 1, x), self._project(f"{prefix}.self_attn", 2, x), mask)
            x = self.norm(f"{prefix}.norm2", queries)
            queries = queries + self._attention(
                f"{prefix}.multihead_attn", self._project(f"{prefix}.multihead_attn", 0, x), keys, values, None)
            x = self.norm(f"{prefix}.norm3", queries)
            queries = queries + self.linear(f"{prefix}.linear2", nn.gelu(self.linear(f"{prefix}.linear1", x)))
        return queries

    # ---- readouts
    def typed_logits(self, raw: mx.array, routed: mx.array) -> tuple[mx.array, mx.array, mx.array]:
        count, slots = raw.shape[0], cm.DIGIT_SLOTS
        scale = mx.exp(mx.minimum(self.w["lexical_scale"], float(np.log(100.0))))
        unit = _normalize(raw)
        down = (raw @ self.w["slot_down.weight"].T).reshape(count, slots + 2, self.slot_rank)
        digit_prior = (unit @ _normalize(self.w["digit_rows"]).T)[:, None, :]
        month_prior = unit @ _normalize(self.w["month_rows"]).T
        sign_prior = unit @ _normalize(self.w["sign_rows"]).T
        residual = mx.sigmoid(self.w["residual_gate"]) * self.linear("slot_residual", routed)
        digit_residual = residual[:, : slots * 10].reshape(count, slots, 10)
        month_residual, sign_residual = residual[:, slots * 10: slots * 10 + 12], residual[:, slots * 10 + 12:]
        return (
            scale[0] * digit_prior + mx.einsum("nsr,srd->nsd", down[:, :slots], self.w["slot_up"]) + digit_residual,
            scale[1] * month_prior + down[:, slots] @ self.w["month_up"] + month_residual,
            scale[2] * sign_prior + down[:, slots + 1] @ self.w["sign_up"] + sign_residual,
        )

    def typed_outputs(self, raw: mx.array, states: mx.array, routed: mx.array) -> dict[str, mx.array]:
        digits, months, signs = self.typed_logits(raw, routed)
        count = raw.shape[0]
        year_slots = digits[:, cm.YEAR_SLOT: cm.YEAR_SLOT + cm.YEAR_DIGITS]
        return {
            "number": mx.concatenate([signs, self.linear("exponent", states),
                                      digits[:, : cm.NUMBER_DIGITS].reshape(count, -1)], axis=-1),
            "date": mx.concatenate([year_slots.reshape(count, -1), _years(year_slots), months,
                                    _two_digits(digits[:, cm.DAY_SLOT: cm.DAY_SLOT + 2], range(1, 32))], axis=-1),
            "time": mx.concatenate([_two_digits(digits[:, cm.HOUR_SLOT: cm.HOUR_SLOT + 2], range(24)),
                                    _two_digits(digits[:, cm.MINUTE_SLOT: cm.MINUTE_SLOT + 2], range(60)),
                                    _two_digits(digits[:, cm.SECOND_SLOT: cm.SECOND_SLOT + 2], range(60))], axis=-1),
        }

    def group_scores(self, states: mx.array) -> mx.array:
        return self.linear("number_group_up", nn.gelu(self.linear("number_group_down", states))).reshape(
            -1, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)

    def temper_number(self, logits: mx.array, spread: mx.array) -> mx.array:
        classes, digits = cm.EXPONENT_CLASSES, cm.NUMBER_DIGITS
        sign, exponent = logits[:, :2], logits[:, 2: 2 + classes]
        slots = logits[:, 2 + classes:].reshape(-1, digits, 10)
        limit = cm.ESTIMATE_SPREAD_LIMIT
        temperature = mx.exp(limit * mx.tanh(spread / limit))
        slots = mx.concatenate([slots[:, : cm.LEXICAL_DIGITS] / temperature[:, 1:, None], slots[:, cm.LEXICAL_DIGITS:]], axis=1)
        return mx.concatenate([sign, exponent / temperature[:, :1], slots.reshape(slots.shape[0], -1)], axis=-1)

    def forward(self, hidden: mx.array, start: int, stop: int, views: dict[str, np.ndarray],
                memory: list[tuple[mx.array, mx.array]]) -> dict[str, mx.array]:
        """Upstream `forward_lexical`: `hidden` is [tokens, width] in fp32, `views` the index arrays."""
        if any(name.endswith("_prefix_length") for name in views):
            raise ValueError("prefixed number readouts are not supported")
        rows = {name: np.asarray(value) for name, value in views.items()}
        queries = rows["query_positions"]
        item_kinds = [kind for kind in cm.LIST_KINDS if rows[f"{kind}_rows"].shape[0]]
        positions = np.concatenate([queries] + [rows[f"{kind}_items"].reshape(-1) for kind in item_kinds])
        kinds = np.zeros(positions.shape[0], dtype=np.int64)
        for index, group in enumerate(cm.ROUTE_KINDS):
            kinds[rows[f"{group}_rows"]] = index
        offset = queries.shape[0]
        for kind in item_kinds:
            size = rows[f"{kind}_items"].size
            kinds[offset: offset + size] = len(cm.ROUTE_KINDS) + cm.LIST_KINDS.index(kind)
            offset += size

        def take(values: mx.array, index: np.ndarray) -> mx.array:
            return values[mx.array(index.astype(np.int32))]

        raw = take(hidden, positions)
        routed = self._route_layers(
            (self.linear("route_in", self.norm("norm", raw)) + take(self.w["kind_embedding.weight"], kinds))[None], memory)
        routed = self.norm("route_norm", routed[0])
        states = self.norm("norm", raw) + mx.sigmoid(self.w["route_gate"]) * self.linear("route_out", routed)
        count = queries.shape[0]
        output: dict[str, mx.array] = {"null": self.linear("null", states[:count])[:, 0]}
        field_states, field_raw, field_routed = states[:count], raw[:count], routed[:count]
        text, tags, numbers = rows["text_rows"], rows["tags_rows"], rows["number_rows"]
        scale = 1.0 / float(np.sqrt(self.pointer_width))
        tokens = None
        if text.shape[0] or tags.shape[0] or numbers.shape[0]:
            tokens = self.norm("token_norm", hidden[start:stop])
            if text.shape[0] or tags.shape[0]:
                output["trim"] = self.linear("trim", tokens).reshape(-1, 2, cm.TRIM_CLASSES)

        def pointer(query: str, selected: mx.array) -> mx.array:
            query_start, query_end = mx.split(self.linear(query, selected), 2, axis=-1)
            key_start, key_end = mx.split(self.linear("span_key", tokens), 2, axis=-1)
            return _span_scores(query_start @ key_start.T * scale, query_end @ key_end.T * scale, self.w["length_bias"])

        if numbers.shape[0]:
            number_states = take(field_states, numbers)
            output["number_copy"] = pointer("copy_query", number_states)
            output["number_copy_gate"] = self.linear("copy_gate", number_states)[:, 0]
        if text.shape[0]:
            output["text"] = pointer("span_query", take(field_states, text))
        if tags.shape[0]:
            query_begin, query_inside = mx.split(self.linear("tag_query", take(field_states, tags)), 2, axis=-1)
            keys = self.linear("tag_key", tokens)
            begin = query_begin @ keys.T * scale + self.w["tag_bias"][0]
            inside = query_inside @ keys.T * scale + self.w["tag_bias"][1]
            output["tags"] = mx.stack([mx.zeros_like(begin), begin, inside], axis=-1)
        typed_rows = [rows[f"{group}_rows"] for group in ("number", "date", "time")]
        if any(selected.shape[0] for selected in typed_rows):
            selected = np.concatenate(typed_rows)
            typed = self.typed_outputs(take(field_raw, selected), take(field_states, selected), take(field_routed, selected))
            offset = 0
            for group, selected in zip(("number", "date", "time"), typed_rows):
                if selected.shape[0]:
                    output[group] = typed[group][offset: offset + selected.shape[0]]
                offset += selected.shape[0]
            if "number" in output:
                output["number_groups"] = self.group_scores(take(field_states, numbers))
        estimates = rows["estimate_rows"]
        if estimates.shape[0]:
            estimate_states = take(field_states, estimates)
            output["estimate"] = self.linear("bins", estimate_states)
            output["estimate_offset"] = self.linear("offset", estimate_states)
            output["estimate_overflow"] = self.linear("overflow", estimate_states)
            number_logits = self.typed_outputs(take(field_raw, estimates), estimate_states, take(field_routed, estimates))["number"]
            output["estimate_number_raw"] = number_logits
            output["estimate_number"] = self.temper_number(number_logits, self.linear("estimate_spread", estimate_states))
            output["estimate_gate"] = self.linear("estimate_gate", estimate_states)[:, 0]
        offset = count
        for kind in item_kinds:
            shape = rows[f"{kind}_items"].shape
            size = shape[0] * shape[1]
            item_states, item_raw, item_routed = (
                states[offset: offset + size], raw[offset: offset + size], routed[offset: offset + size])
            offset += size
            output[f"{kind}_end"] = self.linear("item_end", item_states)[:, 0].reshape(shape)
            group = "number" if kind == "number_list" else "date"
            output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[group].reshape(*shape, -1)
            if kind == "number_list":
                output["number_list_groups"] = self.group_scores(item_states).reshape(
                    *shape, cm.NUMBER_GROUPS, cm.GROUP_CLASSES)
        return output

    def answer_slot_states(self, hidden: mx.array, field_queries: np.ndarray, kinds: np.ndarray, positions: np.ndarray,
                           lengths: np.ndarray, stop: int, memory: list[tuple[mx.array, mx.array]],
                           chunk: int = 256) -> mx.array:
        """Upstream `answer_slot_states`: each answer-slot position corrected by evidence routed from the prompt."""
        width = positions.shape[1]
        if (width < 1 or positions.shape[0] != field_queries.shape[0] or (lengths < 1).any() or (lengths > width).any()
                or not 0 < stop <= hidden.shape[0]):
            raise ValueError("invalid answer-slot routing layout")
        gate = mx.sigmoid(self.w["route_gate"])
        output = []
        field_batch = max(1, min(chunk // (width + 1), 262144 // stop))
        for first in range(0, positions.shape[0], field_batch):
            last = first + field_batch
            slot_positions = mx.array(positions[first:last].astype(np.int32))
            query_positions = mx.concatenate([mx.array(field_queries[first:last, None].astype(np.int32)), slot_positions], axis=1)
            row_lengths = mx.array(lengths[first:last].astype(np.int32))
            row_kinds = mx.array(kinds[first:last].astype(np.int32))
            padding = mx.arange(width + 1)[None, :] > row_lengths[:, None]
            queries = self.linear("route_in", self.norm("norm", hidden[query_positions]))
            queries = self._route_layers(queries + self.w["kind_embedding.weight"][row_kinds][:, None, :], memory, padding)
            evidence = self.linear("route_out", self.norm("route_norm", queries[:, 1:]))
            scale = mx.tanh(self.w["slot_route_scale"][row_kinds])[:, None, None]
            corrected = hidden[slot_positions] + scale * gate * evidence
            index = mx.minimum(mx.arange(width)[None, :], row_lengths[:, None] - 1)
            output.append(mx.take_along_axis(corrected, index[:, :, None], axis=1))
            mx.eval(output[-1])
        return mx.concatenate(output, axis=0)


# ---------------------------------------------------------------------------------------------- engine
@dataclass
class _Rows:
    """One right-padded batch: token ids, which positions are real, and the rows' image inputs in order."""

    ids: np.ndarray
    valid: np.ndarray
    media: list[dict[str, Any]] = field(default_factory=list)

    @property
    def lengths(self) -> np.ndarray:
        return self.valid.sum(axis=1)


def _common_prefix(sequences: list[list[int]]) -> int:
    first, last = min(sequences), max(sequences)
    return next((i for i, (a, b) in enumerate(zip(first, last)) if a != b), min(len(first), len(last)))


class MLXEngine:
    """Share one MLX backbone between grammar-constrained extraction and typed readouts."""

    def __init__(self, path: Path, tokenizer: Any, *, batch_size: int = 8, batch_token_budget: int = 8192,
                 rope: str = "temporal", device: str = "gpu", memory_limit_gb: float | None = None) -> None:
        import xgrammar as xgr

        if rope not in ROPE_MODES:
            raise ValueError(f"rope must be one of {ROPE_MODES}")
        self.path, self.tokenizer, self.rope = Path(path), tokenizer, rope
        self.batch_size, self.batch_token_budget = max(1, int(batch_size)), max(1, int(batch_token_budget))
        self.text_batch_size = self.batch_size
        self.device = device  # "gpu", or "cpu" for bit-accurate float32 checks (slow)
        self.capture: Callable[[str, Any], None] | None = None  # test hook: receives intermediate arrays
        # A request that would need more memory than this fails with an error instead of exhausting the machine.
        self.memory_cap = int((memory_limit_gb * 1e9) if memory_limit_gb else MEMORY_FRACTION * mx.device_info()["memory_size"])
        self._worker = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="clairic-model")
        self._features: dict[str, mx.array] = {}
        self._prefixes: dict[tuple, Cache] = {}
        self.prefix_cache = True  # read a request's images once and continue every prompt from that state
        # Every MLX call happens on the worker thread, including loading the weights.
        self._worker.submit(self._load).result()
        self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
        self.pad = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else self.eos
        self.vocab = self.lm.lm_head.weight.shape[0]
        info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=self.vocab, stop_token_ids=[self.eos])
        self.compiler = xgr.GrammarCompiler(info, max_threads=2)
        self._xgr = xgr

    # ---- loading
    def _load(self) -> None:
        path = self.path
        if self.device == "cpu":
            mx.set_default_device(mx.cpu)
        config = json.loads((path / "config.json").read_text())
        self.model = Model(ModelConfig.from_dict(config))
        self.lm = self.model.language_model
        text = self.lm.model
        hidden, eps = config["text_config"]["hidden_size"], config["text_config"]["rms_norm_eps"]
        text.norm = PreciseRMSNorm(hidden, eps)
        for layer in text.layers:
            layer.input_layernorm = PreciseRMSNorm(hidden, eps)
            layer.post_attention_layernorm = PreciseRMSNorm(hidden, eps)
            if not layer.is_linear:
                layer.self_attn.q_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
                layer.self_attn.k_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps)
        weights: dict[str, mx.array] = {}
        for shard in sorted(path.glob("model*.safetensors")):
            weights.update(mx.load(str(shard)))
        if quantization := config.get("quantization"):
            def quantized(path: str, module: nn.Module) -> bool | dict:
                if not hasattr(module, "to_quantized") or f"{path}.scales" not in weights:
                    return False
                settings = quantization.get(path)  # modules with their own bit width
                return settings if isinstance(settings, dict) else True

            nn.quantize(self.model, group_size=quantization["group_size"], bits=quantization["bits"],
                        mode=quantization.get("mode", "affine"), class_predicate=quantized)
        self.model.load_weights(list(weights.items()), strict=True)
        self.model.eval()
        mx.eval(self.model.parameters())
        self.image_token = int(config["image_token_id"])
        self.vision_end = int(config["vision_end_token_id"])
        self.dtype = text.layers[0].linear_attn.conv1d.weight.dtype  # never quantized: bf16 (release) or fp32
        self.backbone = self.model
        self.head = TypedHead(path, round_bf16=self.dtype != mx.float32)

    # ---- request scope (image features are kept for one request)
    def begin_request(self) -> None:
        self._features, self._prefixes = {}, {}

    async def end_request(self) -> None:
        await asyncio.get_running_loop().run_in_executor(self._worker, self._clear)

    def _clear(self) -> None:
        self._features, self._prefixes = {}, {}
        mx.clear_cache()

    def shutdown(self) -> None:
        self._worker.shutdown(wait=True, cancel_futures=True)
        self._features, self._prefixes = {}, {}

    def _check_memory(self) -> None:
        if mx.get_active_memory() > self.memory_cap:
            mx.clear_cache()
            if mx.get_active_memory() > self.memory_cap:
                raise MemoryError(
                    f"this request needs more than the {self.memory_cap / 1e9:.0f} GB memory limit; "
                    "shorten the input, lower batch_token_budget, or raise memory_limit_gb")

    # ---- backbone
    def _image_features(self, mapping: dict[str, Any]) -> mx.array:
        key = mapping.get("clair_pixels_key")
        if key is None:
            pixels = np.asarray(mapping["pixel_values"])
            key = hashlib.sha1(pixels.tobytes() + np.asarray(mapping["image_grid_thw"]).tobytes()).hexdigest()
        if key not in self._features:
            tower = self.model.vision_tower
            pixels = mx.array(mapping["pixel_values"]).astype(tower.patch_embed.proj.weight.dtype)
            features, _ = tower(pixels, mx.array(np.asarray(mapping["image_grid_thw"])))
            mx.eval(features)
            self._features[key] = features
        return self._features[key]

    def _rows(self, sequences: list[list[int]], media: list[Any]) -> _Rows:
        width = max(map(len, sequences))
        ids = np.full((len(sequences), width), self.pad, dtype=np.int32)
        valid = np.zeros((len(sequences), width), dtype=bool)
        for row, sequence in enumerate(sequences):
            ids[row, : len(sequence)] = sequence
            valid[row, : len(sequence)] = True
        return _Rows(ids, valid, [mapping for payload in media if payload for mapping in payload])

    def _embed(self, rows: _Rows) -> tuple[mx.array, mx.array]:
        """Input embeddings (image features merged in) and rotary positions [3, B, L] of a right-padded batch."""
        tokens, mask = mx.array(rows.ids), mx.array(rows.valid.astype(np.int32))
        if rows.media:
            features = mx.concatenate([self._image_features(mapping) for mapping in rows.media], axis=0)
            grids = mx.array(np.concatenate([np.asarray(mapping["image_grid_thw"]) for mapping in rows.media]))
            embedded = self.model.get_input_embeddings(
                tokens, features, image_grid_thw=grids, mask=mask, cached_image_features=features)
        else:
            embedded = self.model.get_input_embeddings(tokens, None, mask=mask)
        positions = embedded.position_ids
        if positions.ndim == 2:  # text only: the same position on the three rotary axes
            positions = mx.broadcast_to(positions[None], (3, *positions.shape))
        return embedded.inputs_embeds, positions

    def _rotary(self, positions: mx.array) -> mx.array:
        # "temporal" (default) is what the upstream runtime does in bfloat16, its serving configuration: the
        # rotary table is indexed by the first (temporal) position axis for every frequency, so the tokens of an
        # image share one rotary position. "mrope" is the stock Transformers 3-axis rotary, which upstream runs
        # in float32. Text tokens are identical in both.
        if self.rope == "temporal":
            return mx.broadcast_to(positions[:1], positions.shape)
        return positions

    def _linear(self, attention: nn.Module, x: mx.array, valid: mx.array | None, window: mx.array | None,
                memory: mx.array | None, lengths: mx.array | None = None) -> tuple[mx.array, mx.array, mx.array]:
        """A Gated DeltaNet layer continuing from a convolution window and recurrent memory (padding skipped)."""
        batch, length, _ = x.shape
        mixed = attention.in_proj_qkv(x)
        z = attention.in_proj_z(x).reshape(batch, length, -1, attention.head_v_dim)
        b, a = attention.in_proj_b(x), attention.in_proj_a(x)
        if valid is not None:
            mixed = mx.where(valid[..., None], mixed, 0)
        if window is None:
            window = mx.zeros((batch, attention.conv_kernel_size - 1, attention.conv_dim), dtype=mixed.dtype)
        conv_input = mx.concatenate([window.astype(mixed.dtype), mixed], axis=1)
        conv = nn.silu(attention.conv1d(conv_input))
        q, k, v = (
            t.reshape(batch, length, heads, width)
            for t, heads, width in zip(
                mx.split(conv, [attention.key_dim, 2 * attention.key_dim], -1),
                [attention.num_k_heads, attention.num_k_heads, attention.num_v_heads],
                [attention.head_k_dim, attention.head_k_dim, attention.head_v_dim], strict=True)
        )
        q, k = attention._normalize_qk(q, k)
        out, memory = gated_delta_update(q, k, v, a, b, attention.A_log, attention.dt_bias, state=memory, mask=valid,
                                         use_kernel=True)
        out = attention.out_proj(attention.norm(out, z).reshape(batch, length, -1))
        taps = attention.conv_kernel_size - 1
        if lengths is None:
            return out, memory, conv_input[:, -taps:]
        # Right-padded rows: keep the last real inputs of each row, not the padding after them.
        index = lengths[:, None] + mx.arange(taps)[None, :]
        index = mx.broadcast_to(index[:, :, None], (batch, taps, conv_input.shape[-1]))
        return out, memory, mx.take_along_axis(conv_input, index, axis=1)

    def _attention(self, attention: nn.Module, x: mx.array, positions: mx.array, keys: mx.array | None,
                   values: mx.array | None, valid: mx.array | None, offset: int) -> tuple[mx.array, mx.array, mx.array]:
        """Causal attention of new tokens over the cached and new keys; padded keys are masked."""
        batch, length, _ = x.shape
        queries, new_keys, new_values, gate, _ = attention._prepare_projected_qkv(
            attention.q_proj(x), attention.k_proj(x), attention.v_proj(x), None, positions, None, None)
        if keys is not None:
            new_keys = mx.concatenate([keys, new_keys], axis=2)
            new_values = mx.concatenate([values, new_values], axis=2)
        total = new_keys.shape[2]
        # The score matrix is queries x keys per head. Long inputs are attended in slices of query rows so its
        # size stays bounded (a slice's output depends only on its own rows); unsliced, a 40k-token input needs
        # tens of GB per layer.
        rows = max(1, ATTENTION_ELEMENTS // (batch * attention.num_attention_heads * total))
        outputs = []
        for start in range(0, length, rows):
            stop = min(length, start + rows)
            mask = None
            if length > 1:
                key_index = mx.arange(total)[None, :]
                row_index = mx.arange(offset + start, offset + stop)[:, None]
                mask = (key_index <= row_index)[None, None]
                if valid is not None:  # a padded query still sees itself, so no row is fully masked
                    mask = mask & (valid[:, None, None, :] | (key_index == row_index)[None, None])
            elif valid is not None:
                mask = valid[:, None, None, :]
            outputs.append(mx.fast.scaled_dot_product_attention(
                queries[:, :, start:stop], new_keys, new_values, scale=attention.scale, mask=mask))
            if rows < length:
                mx.eval(outputs[-1])
                self._check_memory()
        output = outputs[0] if len(outputs) == 1 else mx.concatenate(outputs, axis=2)
        output = output.transpose(0, 2, 1, 3).reshape(batch, length, -1)
        return attention.o_proj(output * mx.sigmoid(gate)), new_keys, new_values

    def _forward(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
                 keep: bool = True) -> tuple[mx.array, Cache | None]:
        """Run the layers over new tokens, optionally continuing from a cache.

        `lengths` gives each row's number of real tokens when the batch is right-padded (None: no padding).
        Returns the hidden states before the final norm and, with `keep`, the cache after each row's last real
        token (padded positions stay in the attention cache, marked invalid).
        """
        layers = self.lm.model.layers
        cache = cache or Cache.empty(len(layers))
        batch, length, _ = h.shape
        valid = row_lengths = None
        if lengths is not None and (lengths < length).any():
            row_lengths = mx.array(lengths.astype(np.int32))
            valid = mx.arange(length)[None, :] < row_lengths[:, None]
        all_valid = None
        if cache.valid is not None or valid is not None:
            before = cache.valid if cache.valid is not None else mx.ones((batch, cache.length), dtype=mx.bool_)
            new = valid if valid is not None else mx.ones((batch, length), dtype=mx.bool_)
            all_valid = mx.concatenate([before, new], axis=1)
        rotary = self._rotary(positions)
        updated = Cache.empty(len(layers)) if keep else None
        stepwise = batch * length > 4096  # keep the lazy graph small on long inputs
        for index, layer in enumerate(layers):
            x = layer.input_layernorm(h)
            if layer.is_linear:
                r, memory, window = self._linear(layer.linear_attn, x, valid, cache.window[index], cache.memory[index],
                                                 row_lengths)
                if keep:
                    updated.memory[index], updated.window[index] = memory, window
            else:
                r, keys, values = self._attention(layer.self_attn, x, rotary, cache.keys[index], cache.values[index],
                                                  all_valid, cache.length)
                if keep:
                    updated.keys[index], updated.values[index] = keys, values
            h = h + r
            h = h + layer.mlp(layer.post_attention_layernorm(h))
            if stepwise:
                mx.eval(h, *(updated.arrays() if keep else []))
                self._check_memory()
        if keep:
            updated.valid, updated.length = all_valid, cache.length + length
        return h, updated

    def _run(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None,
             keep: bool = True) -> tuple[mx.array, Cache | None]:
        """`_forward` over a whole input. Long inputs go through the layers in segments, each continuing from
        the cache of the previous one, so working memory depends on the segment size rather than the input
        length (attention is causal and the linear layers are recurrent, so the result is the same). With
        `keep`, the returned cache has `position` set to each row's next rotary position."""
        batch, length, _ = h.shape
        segment = max(256, SEGMENT_TOKENS // batch)
        if length <= segment:
            out, cache = self._forward(h, positions, lengths, cache, keep)
        else:
            outputs = []
            for start in range(0, length, segment):
                stop = min(length, start + segment)
                inside = None if lengths is None else np.clip(lengths - start, 0, stop - start)
                part, cache = self._forward(h[:, start:stop], positions[:, :, start:stop], inside, cache)
                mx.eval(part, *cache.arrays())
                self._check_memory()
                outputs.append(part)
            out = mx.concatenate(outputs, axis=1)
            cache = cache if keep else None
        if keep:
            last = mx.array((lengths if lengths is not None else np.full(batch, length)).astype(np.int32)) - 1
            index = mx.broadcast_to(last[None, :, None], (positions.shape[0], batch, 1))
            cache.position = mx.take_along_axis(positions, index, axis=2)[:, :, 0].max(axis=0) + 1
        return out, cache

    def _hidden(self, sequences: list[list[int]], media: list[Any]) -> mx.array:
        """Final-norm hidden states of an unpadded single sequence."""
        rows = self._rows(sequences, media)
        h, positions = self._embed(rows)
        h, _ = self._run(h, positions, None, None, keep=False)
        return self.lm.model.norm(h)

    # ---- typed readout (upstream `read_record`)
    def _read_record(self, states: mx.array, item: Any) -> dict[str, np.ndarray]:
        start, length, kinds, queries, items, slot_fields, slots = cm.parse_prediction_layout(
            cm.prediction_layout(item), len(item.input_ids), with_slots=True)
        views = cm.prediction_views(kinds, queries, items, None)
        hidden = states.astype(mx.float32)
        stop = start + length
        memory = self.head._memory(hidden, stop)
        output = self.head.forward(hidden, start, stop, views, memory)
        mx.eval(output)
        values: dict[str, np.ndarray] = {name: np.array(value) for name, value in output.items()}
        if slots:
            width = max(map(len, slots))
            positions = np.array([row + [row[-1]] * (width - len(row)) for row in slots], dtype=np.int64)
            field_queries = np.asarray(views["query_positions"])[slot_fields]
            slot_kinds = np.array([cm.ROUTE_KINDS.index("tags" if kinds[row] == "text_list" else kinds[row])
                                   for row in slot_fields], dtype=np.int64)
            lengths = np.array([len(row) for row in slots], dtype=np.int64)
            slot_states = self.head.answer_slot_states(hidden, field_queries, slot_kinds, positions, lengths, stop, memory)
            slot_states = slot_states.reshape(-1, slot_states.shape[-1]).astype(states.dtype)
            ids, logprobs = [], []
            for first in range(0, slot_states.shape[0], 256):
                logits = self.lm.lm_head(slot_states[first: first + 256]).astype(mx.float32)
                best = mx.argmax(logits, axis=-1)
                ids.append(best)
                logprobs.append(mx.take_along_axis(logits, best[:, None], axis=-1)[:, 0] - mx.logsumexp(logits, axis=-1))
                mx.eval(ids[-1], logprobs[-1])
            values["slot_ids"] = np.array(mx.concatenate(ids)).reshape(positions.shape)
            values["slot_log_probs"] = np.array(mx.concatenate(logprobs)).reshape(positions.shape)
        return values

    def _read_typed(self, item: Any, media: Any, _cancelled: threading.Event | None = None) -> dict[str, dict[str, Any]]:
        if _cancelled is not None and _cancelled.is_set():
            raise RuntimeError("Inference cancelled")
        states = self._hidden([list(item.input_ids)], [media])[0]
        mx.eval(states)
        if _cancelled is not None and _cancelled.is_set():
            raise RuntimeError("Inference cancelled")
        values = self._read_record(states, item)
        compact = cm.compact_output(values)
        if self.capture is not None:
            self.capture("hidden", np.array(states.astype(mx.float32)))
            self.capture("compact", np.asarray(compact, dtype=np.float32))
            if "slot_ids" in values:
                self.capture("slots", values["slot_ids"])
            if media:
                self.capture("pixels", np.asarray(media[0]["pixel_values"]).astype(np.float16))
                self.capture("grid", np.asarray(media[0]["image_grid_thw"]))
        values = cm.expand_output(compact, item, bins=self.head.bins, estimate_lexical=True, span_trim=True,
                                  number_groups=True, number_pointer=True)
        return cm.decode_record(values, item, self.tokenizer)

    async def read_typed(self, item: Any, media: Any = None) -> dict[str, dict[str, Any]]:
        return await self._submit(lambda cancelled: self._read_typed(item, media, _cancelled=cancelled))

    # ---- constrained generation
    def _grammar(self, parameters: dict[str, Any]) -> Any:
        if "json_schema" in parameters:
            value = parameters["json_schema"]
            return self.compiler.compile_json_schema(json.loads(value) if isinstance(value, str) else value,
                                                     any_whitespace=True)
        if "regex" in parameters:
            return self.compiler.compile_regex(parameters["regex"])
        if "ebnf" in parameters:
            return self.compiler.compile_grammar(parameters["ebnf"])
        raise ValueError("Extraction continuations require a JSON schema, regex, or grammar")

    def _logits(self, h: mx.array) -> np.ndarray:
        """Vocabulary logits (fp32) from hidden states [B, 1, width] before the final norm."""
        return np.array(self.lm.lm_head(self.lm.model.norm(h))[:, 0].astype(mx.float32))

    @staticmethod
    def _media_key(payload: Any) -> tuple | None:
        return tuple(mapping.get("clair_pixels_key") or id(mapping["pixel_values"]) for mapping in payload) if payload else None

    def _image_prefix(self, sequences: list[list[int]], media: list[Any]) -> tuple[Cache | None, int]:
        """The state after the images shared by every prompt of a batch, computed once per request.

        Image prompts start with the same image tokens and differ only afterwards. The shared part, up to the
        last `<|vision_end|>` common to all rows, is run once and each row continues from its cache; attention
        is causal and the linear layers are recurrent, so this equals running every full prompt.
        """
        if not self.prefix_cache or not media[0] or any(self._media_key(m) != self._media_key(media[0]) for m in media):
            return None, 0
        common = sequences[0][: _common_prefix(sequences)]
        if self.vision_end not in common:
            return None, 0
        cut = len(common) - common[::-1].index(self.vision_end)
        if any(self.image_token in sequence[cut:] for sequence in sequences):
            return None, 0
        key = (self._media_key(media[0]), hashlib.sha1(np.asarray(common[:cut], dtype=np.int32).tobytes()).hexdigest())
        if key not in self._prefixes:
            rows = self._rows([common[:cut]], [media[0]])
            h, positions = self._embed(rows)
            _, cache = self._run(h, positions, None, None)
            mx.eval(cache.position, *cache.arrays())
            self._prefixes[key] = cache
        return self._prefixes[key], cut

    def _prefill(self, sequences: list[list[int]], media: list[Any]) -> tuple[mx.array, Cache]:
        """Run the prompts of one batch. Returns each row's last hidden state [B, 1, width] and the cache."""
        size = len(sequences)
        prefix, cut = self._image_prefix(sequences, media)
        if prefix is None:
            rows = self._rows(sequences, media)
            h, positions = self._embed(rows)
            cache = None
        else:
            rows = self._rows([sequence[cut:] for sequence in sequences], [None] * size)
            h = self.lm.model.embed_tokens(mx.array(rows.ids))
            positions = prefix.position[0] + mx.arange(rows.ids.shape[1])
            positions = mx.broadcast_to(positions[None, None, :], (3, size, rows.ids.shape[1]))

            def spread(array: mx.array | None) -> mx.array | None:
                return None if array is None else mx.broadcast_to(array, (size, *array.shape[1:]))

            cache = Cache([spread(a) for a in prefix.keys], [spread(a) for a in prefix.values],
                          [spread(a) for a in prefix.memory], [spread(a) for a in prefix.window], None, prefix.length)
        lengths = rows.lengths
        h, cache = self._run(h, positions, lengths, cache)
        last = mx.array((lengths - 1).astype(np.int32))
        index = mx.broadcast_to(last[:, None, None], (size, 1, h.shape[-1]))
        return mx.take_along_axis(h, index, axis=1), cache

    def _decode(self, sequences: list[list[int]], media: list[Any], grammars: list[Any], maximum: int,
                stop_ids: tuple[int, ...], score_ids: list[int], cancelled: threading.Event | None,
                ) -> list[tuple[list[int], list[float], list[float]]]:
        """Greedy decoding of one batch under per-row grammars. Returns (tokens, log-probabilities, first-step
        log-probabilities of `score_ids`) per row; tokens end with the stop token when one was produced."""
        size = len(sequences)
        h, cache = self._prefill(sequences, media)
        mx.eval(h, cache.position, *cache.arrays())
        scores = self._logits(h)
        matchers = [self._xgr.GrammarMatcher(grammar) for grammar in grammars]
        bitmask = np.zeros(self._xgr.get_bitmask_shape(size, self.vocab), dtype=np.int32)
        tokens: list[list[int]] = [[] for _ in range(size)]
        logprobs: list[list[float]] = [[] for _ in range(size)]
        first: list[list[float]] = [[] for _ in range(size)]
        done = [False] * size
        for step in range(maximum):
            for row, matcher in enumerate(matchers):
                if not matcher.is_terminated():
                    matcher.fill_next_token_bitmask(bitmask, row)
            allowed = np.unpackbits(bitmask.view(np.uint8), axis=-1, bitorder="little")[:, : self.vocab].astype(bool)
            masked = np.where(allowed, scores, -np.inf).astype(np.float32)
            peak = masked.max(axis=-1, keepdims=True)
            with np.errstate(divide="ignore"):
                log_probabilities = masked - peak - np.log(np.exp(masked - peak).sum(axis=-1, keepdims=True, dtype=np.float32))
            choices = masked.argmax(axis=-1)
            for row in range(size):
                if step == 0:
                    first[row] = [float(log_probabilities[row, token]) for token in score_ids]
                if done[row]:
                    continue
                token = int(choices[row])
                tokens[row].append(token)
                logprobs[row].append(float(log_probabilities[row, token]))
                if token in stop_ids:
                    done[row] = True
                if not matchers[row].is_terminated() and not matchers[row].accept_token(token):
                    raise RuntimeError("Generated token rejected by its grammar")
            if all(done) or step + 1 == maximum:
                break
            if cancelled is not None and cancelled.is_set():
                raise RuntimeError("Inference cancelled")
            feed = mx.array([[self.pad if done[row] else tokens[row][-1]] for row in range(size)])
            position = cache.position
            h, cache = self._forward(self.lm.model.embed_tokens(feed), mx.broadcast_to(position[None, :, None], (3, size, 1)),
                                     None, cache)
            cache.position = position + 1
            mx.eval(h, cache.position, *cache.arrays())
            scores = self._logits(h)
        return list(zip(tokens, logprobs, first, strict=True))

    def _generate(self, *, input_ids: list[int] | list[list[int]], image_data: Any = None,
                  sampling_params: dict[str, Any] | list[dict[str, Any]], rid: Any = None,
                  token_ids_logprob: list[int] | None = None, _cancelled: threading.Event | None = None,
                  **kwargs: Any) -> Any:
        if _cancelled is not None and _cancelled.is_set():
            raise RuntimeError("Inference cancelled")
        if not input_ids:
            return []
        single = isinstance(input_ids[0], int)
        sequences = [input_ids] if single else input_ids
        size = len(sequences)
        parameters = [sampling_params] * size if isinstance(sampling_params, dict) else sampling_params
        media = [image_data] if single else (image_data or [None] * size)
        ids = [rid] if single else (rid or [None] * size)
        if not (len(parameters) == len(media) == len(ids) == size):
            raise ValueError("Extraction batch arguments have inconsistent lengths")
        results: list[Any] = [None] * size
        groups: dict[tuple[int, tuple[int, ...]], list[int]] = defaultdict(list)
        for index, params in enumerate(parameters):
            if params.get("temperature", 0) != 0:
                raise ValueError("Clairic extraction uses deterministic decoding")
            groups[(int(params.get("max_new_tokens", 256)), tuple(params.get("stop_token_ids", [self.eos])))].append(index)
        for (maximum, stop_ids), indices in groups.items():
            # Rows with the same images and similar length share a batch; a row's result does not depend on
            # its batch. Rows that share images are charged for the image prefix once.
            indices = sorted(indices, key=lambda index: (str(self._media_key(media[index])), len(sequences[index])))
            chunks: list[list[int]] = []
            pending: list[int] = []
            for index in indices:
                if pending and (len(pending) >= self.batch_size
                                or self._media_key(media[index]) != self._media_key(media[pending[0]])
                                or self._cost([sequences[i] for i in (*pending, index)], media[index]) > self.batch_token_budget):
                    chunks.append(pending)
                    pending = []
                pending.append(index)
            if pending:
                chunks.append(pending)
            for chunk in chunks:
                if _cancelled is not None and _cancelled.is_set():
                    raise RuntimeError("Inference cancelled")
                decoded = self._decode([sequences[i] for i in chunk], [media[i] for i in chunk],
                                       [self._grammar(parameters[i]) for i in chunk], maximum, stop_ids,
                                       list(token_ids_logprob or []), _cancelled)
                for index, (output, logprobs, first) in zip(chunk, decoded, strict=True):
                    stopped = bool(output) and output[-1] in stop_ids
                    results[index] = {
                        "output_ids": output,
                        "text": self.tokenizer.decode(output[:-1] if stopped else output, skip_special_tokens=False,
                                                      clean_up_tokenization_spaces=False),
                        "meta_info": {
                            "id": ids[index],
                            "completion_tokens": len(output),
                            "finish_reason": {"type": "stop", "matched": output[-1]} if stopped else {"type": "length"},
                            "output_token_logprobs": [(value, token, None) for value, token in zip(logprobs, output, strict=True)],
                            "output_token_ids_logprobs": [[(value, token, None) for value, token in
                                                           zip(first, token_ids_logprob or [], strict=True)]],
                        },
                    }
        return results[0] if single else results

    def _cost(self, sequences: list[list[int]], payload: Any) -> int:
        """Tokens a batch runs through the model: the padded width times its rows (shared image tokens once)."""
        shared = 0
        if payload and self.prefix_cache and len(sequences) > 1:
            common = sequences[0][: _common_prefix(sequences)]
            shared = len(common) - common[::-1].index(self.vision_end) if self.vision_end in common else 0
        return (max(map(len, sequences)) - shared) * len(sequences)

    async def async_generate(self, **kwargs: Any) -> Any:
        return await self._submit(lambda cancelled: self._generate(_cancelled=cancelled, **kwargs))

    async def _submit(self, operation: Callable[[threading.Event], Any]) -> Any:
        cancelled = threading.Event()
        future = asyncio.get_running_loop().run_in_executor(self._worker, lambda: operation(cancelled))
        try:
            return await asyncio.shield(future)
        except asyncio.CancelledError:
            cancelled.set()
            # Admission stays occupied until the worker has actually stopped.
            while not future.done():
                try:
                    await asyncio.shield(future)
                except asyncio.CancelledError:
                    continue
                except Exception:
                    break
            if not future.cancelled():
                future.exception()
            raise