areneau commited on
Commit
24191a8
·
verified ·
1 Parent(s): dc7d8aa

Update Clairic Transformers runtime

Browse files
README.md CHANGED
@@ -35,9 +35,9 @@ Use Python 3.11 or later and install the packages listed in `requirements.txt`.
35
  Transformers; `xgrammar` supplies schema constraints through a Transformers logits
36
  processor.
37
 
38
- The supported GPU setup is Linux x86_64 with an H100 or H200, PyTorch 2.11.0
39
  with CUDA 13.0, and Transformers 5.10.2. The CUDA BF16 default uses FlashAttention 3
40
- for text and vision, with batches of up to eight sequences. The first load fetches
41
  the pinned attention kernel through `kernels`; cache it before running offline.
42
  Explicit `attn_implementation`, `batch_size`, and `text_batch_size` loader options
43
  remain available.
 
35
  Transformers; `xgrammar` supplies schema constraints through a Transformers logits
36
  processor.
37
 
38
+ The supported GPU setup is Linux x86_64 with an H100 or H200, PyTorch 2.13.0
39
  with CUDA 13.0, and Transformers 5.10.2. The CUDA BF16 default uses FlashAttention 3
40
+ for text and vision, with batches of up to 64 sequences. The first load fetches
41
  the pinned attention kernel through `kernels`; cache it before running offline.
42
  Explicit `attn_implementation`, `batch_size`, and `text_batch_size` loader options
43
  remain available.
clairic_runtime/attention_metadata.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Reuse pinned FA3 unpadding metadata within one owned decode forward."""
2
+ from __future__ import annotations
3
+
4
+ from collections.abc import Callable, Iterator
5
+ from contextlib import contextmanager
6
+ from contextvars import ContextVar
7
+ import threading
8
+ from typing import Any
9
+
10
+ import torch
11
+ from transformers.integrations.flash_attention import flash_attention_forward
12
+ from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis
13
+
14
+ _PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0"
15
+ _VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"})
16
+
17
+
18
+ class _Entry:
19
+ def __init__(self) -> None:
20
+ self.key: tuple[Any, ...] | None = None
21
+ self.mask: torch.Tensor | None = None
22
+ self.indices: torch.Tensor | None = None
23
+ self.cu_k: torch.Tensor | None = None
24
+ self.cu_q: torch.Tensor | None = None
25
+ self.maximum = 0
26
+ self.bytes = 0
27
+
28
+ def clear(self) -> None:
29
+ self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None
30
+ self.maximum, self.bytes = 0, 0
31
+
32
+
33
+ class DecodeMetadata:
34
+ def __init__(self, max_bytes: int = 8 * 1024**2) -> None:
35
+ if max_bytes <= 0:
36
+ raise ValueError("Expected a positive metadata bound")
37
+ self.max_bytes = max_bytes
38
+ self._lock = threading.Lock()
39
+ self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None)
40
+ self._active: _Entry | None = None
41
+ self._closed = False
42
+ self.scopes = 0
43
+ self.materializations = 0
44
+ self.hits = 0
45
+ self.peak_bytes = 0
46
+
47
+ @contextmanager
48
+ def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]:
49
+ token = self._current.set(None)
50
+ acquired = False
51
+ entry = None
52
+ try:
53
+ inputs = kwargs.get("input_ids", args[0] if args else None)
54
+ if inputs is None:
55
+ inputs = kwargs.get("inputs_embeds")
56
+ eligible = (not self._closed and not torch.is_grad_enabled()
57
+ and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling()
58
+ and isinstance(inputs, torch.Tensor) and inputs.is_cuda
59
+ and inputs.ndim in (2, 3) and inputs.shape[1] == 1)
60
+ if eligible and self._lock.acquire(blocking=False):
61
+ acquired = True
62
+ if not self._closed:
63
+ entry = _Entry()
64
+ self._active = entry
65
+ self._current.set(entry)
66
+ self.scopes += 1
67
+ yield
68
+ finally:
69
+ if entry is not None:
70
+ entry.clear()
71
+ self._active = None
72
+ self._current.reset(token)
73
+ if acquired:
74
+ self._lock.release()
75
+
76
+ def _metadata(self, mask: torch.Tensor) -> _Entry | None:
77
+ entry = self._current.get()
78
+ if self._closed or entry is None:
79
+ return None
80
+ batch = mask.shape[0]
81
+ if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes:
82
+ entry.clear()
83
+ return None
84
+ try:
85
+ version = mask._version
86
+ except RuntimeError:
87
+ version = None
88
+ key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device,
89
+ mask.dtype, tuple(mask.shape), tuple(mask.stride()),
90
+ torch.cuda.current_stream(mask.device).cuda_stream)
91
+ if entry.mask is mask and entry.key == key:
92
+ self.hits += 1
93
+ return entry
94
+ entry.clear()
95
+ indices, cu_k, maximum = _get_unpad_data(mask)
96
+ cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device)
97
+ retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q))
98
+ if retained > self.max_bytes:
99
+ return None
100
+ entry.key, entry.mask = key, mask
101
+ entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q
102
+ entry.maximum, entry.bytes = int(maximum.item()), retained
103
+ self.materializations += 1
104
+ self.peak_bytes = max(self.peak_bytes, retained)
105
+ return entry
106
+
107
+ def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]],
108
+ module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
109
+ mask: torch.Tensor | None, *, dropout: float, scaling: float,
110
+ **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
111
+ eligible = (not self._closed and self._current.get() is not None
112
+ and not torch.is_grad_enabled() and torch.is_inference_mode_enabled()
113
+ and not torch.compiler.is_compiling() and interface is flash_attention_forward
114
+ and module.config._attn_implementation == _PINNED_FA3
115
+ and dropout == 0.0
116
+ and getattr(module.config, "sliding_window", None) is None
117
+ and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys())
118
+ and mask is not None and mask.is_cuda and mask.ndim == 2
119
+ and mask.dtype == torch.bool and mask.numel() > 0
120
+ and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4
121
+ and item.device == mask.device and item.stride(-1) == 1
122
+ for item in (query, key, value)))
123
+ if eligible:
124
+ assert mask is not None
125
+ positions = kwargs.get("position_ids")
126
+ eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0]
127
+ and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1]
128
+ and mask.shape[1] > 1
129
+ and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4
130
+ and query.shape[3] == key.shape[3] == value.shape[3] == 256
131
+ and (positions is None or (isinstance(positions, torch.Tensor)
132
+ and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1))))
133
+ entry = self._metadata(mask) if eligible and mask is not None else None
134
+ if entry is None:
135
+ active = self._current.get()
136
+ if active is not None:
137
+ active.clear()
138
+ return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs)
139
+ assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None
140
+ packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
141
+ packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
142
+ return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling,
143
+ cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k,
144
+ max_length_q=1, max_length_k=entry.maximum, **kwargs)
145
+
146
+ def snapshot(self) -> dict[str, int | bool]:
147
+ return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits,
148
+ "peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0,
149
+ "closed": self._closed}
150
+
151
+ def close(self) -> None:
152
+ self._closed = True
153
+ with self._lock:
154
+ if self._active is not None:
155
+ self._active.clear()
156
+ self._active = None
clairic_runtime/backend.py CHANGED
@@ -40,6 +40,7 @@ from . import (
40
  printed_numbers,
41
  printed_temporal,
42
  receipt_totals,
 
43
  source_dates,
44
  source_rows,
45
  table_rows,
@@ -1113,6 +1114,10 @@ class ClairicFlashModel:
1113
  predictions[name] = {**alternative, "null_recovered": True}
1114
  predictions[name]["tokens"] = tokens
1115
  predictions[name]["retry_input_tokens"] = retry_lengths[name]
 
 
 
 
1116
  return predictions
1117
 
1118
  def null_recovery_record(
 
40
  printed_numbers,
41
  printed_temporal,
42
  receipt_totals,
43
+ row_omission,
44
  source_dates,
45
  source_rows,
46
  table_rows,
 
1114
  predictions[name] = {**alternative, "null_recovered": True}
1115
  predictions[name]["tokens"] = tokens
1116
  predictions[name]["retry_input_tokens"] = retry_lengths[name]
1117
+ if plan := row_omission.prepare(record, predictions):
1118
+ repair, _ = plan
1119
+ alternatives = await self._ar_all(repair, eos)
1120
+ predictions, _ = row_omission.merge(record, predictions, alternatives, plan)
1121
  return predictions
1122
 
1123
  def null_recovery_record(
clairic_runtime/decoder_capture.py ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Owned single-layer graph capture and strict resource lifecycle."""
2
+ from __future__ import annotations
3
+
4
+ import threading
5
+ from typing import Any
6
+ import weakref
7
+
8
+
9
+ def tensor_key(value: Any, *, version: bool) -> tuple[Any, ...]:
10
+ try:
11
+ counter = value._version if version else None
12
+ except RuntimeError:
13
+ counter = None
14
+ storage = value.untyped_storage()
15
+ return (id(value), counter, value.data_ptr(), storage.data_ptr(), storage.nbytes(),
16
+ value.storage_offset(), str(value.device), str(value.dtype),
17
+ tuple(value.shape), tuple(value.stride()))
18
+
19
+
20
+ def callable_key(value: Any) -> tuple[int, int | None]:
21
+ return (id(getattr(value, "__func__", value)),
22
+ id(value.__self__) if hasattr(value, "__self__") else None)
23
+
24
+
25
+ def input_key(value: Any) -> tuple[Any, ...]:
26
+ return str(value.device), str(value.dtype), tuple(value.shape), tuple(value.stride())
27
+
28
+
29
+ def owner_key(torch: Any, owner: Any, cache: Any, operation: Any) -> tuple[Any, ...]:
30
+ modules = []
31
+ for name, module in owner.named_modules():
32
+ if module._forward_hooks or module._forward_pre_hooks:
33
+ raise RuntimeError("Forward hooks invalidate decoder capture")
34
+ scalars = tuple(sorted((key, value) for key, value in vars(module).items()
35
+ if not key.startswith("_") and isinstance(value, (str, bool, int, float))))
36
+ callables = tuple(sorted((key, callable_key(value)) for key, value in vars(module).items()
37
+ if not key.startswith("_") and callable(value)))
38
+ modules.append((name, id(module), callable_key(module.forward), scalars, callables))
39
+ registry = torch.nn.modules.module
40
+ if registry._global_forward_hooks or registry._global_forward_pre_hooks:
41
+ raise RuntimeError("Global forward hooks invalidate decoder capture")
42
+ parameters = tuple((name, tensor_key(value, version=True))
43
+ for name, value in owner.named_parameters())
44
+ if any(item[1][1] is None for item in parameters):
45
+ raise RuntimeError("Versioned parameters required")
46
+ buffers = tuple((name, tensor_key(value, version=True))
47
+ for name, value in owner.named_buffers())
48
+ layer = cache.layers[0]
49
+ if not (cache.has_previous_state(0) and layer.is_conv_states_initialized
50
+ and layer.is_recurrent_states_initialized):
51
+ raise RuntimeError("Already initialized live decode cache required")
52
+ states = tuple(tensor_key(value, version=False)
53
+ for value in (layer.conv_states, layer.recurrent_states))
54
+ return (id(owner), id(cache), id(layer), callable_key(operation), tuple(modules),
55
+ parameters, buffers, states, callable_key(cache.update_recurrent_state),
56
+ callable_key(cache.has_previous_state), callable_key(layer.update_recurrent_state))
57
+
58
+
59
+ def describe_pool(segments: list[dict[str, Any]], pool: tuple[int, int]) -> dict[str, Any]:
60
+ blocks = [block for segment in segments for block in segment["blocks"]]
61
+ pending = {"active_awaiting_free", "active_pending_free"}
62
+ return {"pool": pool, "segments": segments,
63
+ "foreign_pool": any(tuple(segment["segment_pool_id"]) != pool for segment in segments),
64
+ "reserved_bytes": sum(segment["total_size"] for segment in segments),
65
+ "allocated_bytes": sum(segment["allocated_size"] for segment in segments),
66
+ "active_bytes": sum(segment["active_size"] for segment in segments),
67
+ "active_allocated_blocks": sum(block["state"] == "active_allocated" for block in blocks),
68
+ "pending_blocks": sum(block["state"] in pending for block in blocks),
69
+ "unknown_block_states": sorted({block["state"] for block in blocks}
70
+ - pending - {"active_allocated", "inactive"})}
71
+
72
+
73
+ class DecoderGraph:
74
+ """One exact bucket/dtype graph with owned input and cloned FP32 outputs.
75
+
76
+ Capture/replay must run on the engine's exclusive CUDA worker. Mutation is
77
+ intentional only for the two captured cache tensors. Any owner, parameter,
78
+ callable, stream, runtime, cache storage or hook change rejects the session.
79
+ Capture/replay failures propagate; callers must close the engine rather than
80
+ retrying eager work on a possibly invalid CUDA context.
81
+ """
82
+
83
+ def __init__(self, backend: Any, owner: Any, cache: Any, value: Any,
84
+ operation: Any, *, exclusive_process: bool) -> None:
85
+ if not exclusive_process:
86
+ raise ValueError("Exclusive process required")
87
+ self.backend, self.owner, self.cache, self.operation = backend, owner, cache, operation
88
+ self.thread, self.closed = threading.get_ident(), False
89
+ self.context = backend.context(value)
90
+ self.input = input_key(value)
91
+ self.key = owner_key(backend.torch, owner, cache, operation)
92
+ self.entry = backend.capture(value, operation, cache)
93
+ self.check(value)
94
+
95
+ def check(self, value: Any) -> None:
96
+ if self.closed or self.thread != threading.get_ident():
97
+ raise RuntimeError("Closed graph or foreign thread")
98
+ if self.backend.context(value) != self.context or input_key(value) != self.input:
99
+ raise RuntimeError("Input layout, dtype, stream or runtime changed")
100
+ if owner_key(self.backend.torch, self.owner, self.cache, self.operation) != self.key:
101
+ raise RuntimeError("Decoder owner, parameter, callable or cache changed")
102
+
103
+ def run(self, value: Any) -> Any:
104
+ self.check(value)
105
+ return self.direct(value)
106
+
107
+ def direct(self, value: Any) -> Any:
108
+ # The padded manager executes complete guards before direct replay.
109
+ return self.backend.replay_clone(self.entry, value)
110
+
111
+ def close(self) -> dict[str, Any]:
112
+ if self.closed or self.thread != threading.get_ident():
113
+ raise RuntimeError("Closed graph or foreign thread")
114
+ details = self.backend.release(self.entry)
115
+ self.owner = self.cache = self.operation = self.key = None
116
+ self.closed = True
117
+ return details
118
+
119
+
120
+ class TorchBackend:
121
+ def __init__(self, torch: Any, runtime_key: Any) -> None:
122
+ self.torch, self.runtime_key = torch, runtime_key
123
+
124
+ def context(self, value: Any) -> tuple[Any, ...]:
125
+ torch = self.torch
126
+ if (torch.is_grad_enabled() or torch.compiler.is_compiling() or not value.is_cuda
127
+ or torch.cuda.is_current_stream_capturing() or not value.is_contiguous()
128
+ or value.dtype not in (torch.bfloat16, torch.float32)
129
+ or value.ndim != 3 or value.shape[1] != 1 or min(value.shape) < 1):
130
+ raise RuntimeError("Unsupported decoder graph context/input")
131
+ return (torch.cuda.current_stream(value.device).cuda_stream, self.runtime_key())
132
+
133
+ def capture(self, value: Any, operation: Any, cache: Any) -> dict[str, Any]:
134
+ torch = self.torch
135
+ current = torch.cuda.current_stream(value.device)
136
+ side = torch.cuda.Stream(device=value.device)
137
+ side.wait_stream(current)
138
+ layer = cache.layers[0]
139
+ states = (layer.conv_states, layer.recurrent_states)
140
+ saved = tuple(state.clone(memory_format=torch.preserve_format) for state in states)
141
+ side.wait_stream(current)
142
+
143
+ def restore(current_states: Any, before_states: Any) -> None:
144
+ for state, before in zip(current_states, before_states, strict=True):
145
+ state.copy_(before)
146
+
147
+ graph = torch.cuda.CUDAGraph()
148
+ with torch.cuda.stream(side):
149
+ static_input = torch.empty_like(value)
150
+ static_input.copy_(value)
151
+ for _ in range(2):
152
+ warm_output = operation(static_input)
153
+ side.synchronize()
154
+ del warm_output
155
+ restore(states, saved)
156
+ side.synchronize()
157
+ graph.capture_begin(capture_error_mode="global")
158
+ try:
159
+ static_output = operation(static_input)
160
+ finally:
161
+ graph.capture_end()
162
+ side.synchronize()
163
+ restore(states, saved)
164
+ current.wait_stream(side)
165
+ current.synchronize()
166
+ assert static_output.dtype == torch.float32
167
+ static_input.record_stream(current)
168
+ static_output.record_stream(current)
169
+ for state, before in zip(states, saved, strict=True):
170
+ assert torch.equal(state.contiguous().view(torch.int16), before.contiguous().view(torch.int16))
171
+ del saved, states
172
+ pool = tuple(graph.pool())
173
+ details = describe_pool(torch.cuda.memory_snapshot(mempool_id=pool, include_traces=False), pool)
174
+ assert not details["foreign_pool"] and details["reserved_bytes"] > 0
175
+ return {"graph": graph, "static_input": static_input, "static_output": static_output,
176
+ "stream": current, "side": side, "pool": pool,
177
+ "pool_at_capture": details,
178
+ "static_input_storage_bytes": static_input.untyped_storage().nbytes(),
179
+ "capture_mutation_restored": True}
180
+
181
+ def replay_clone(self, entry: dict[str, Any], value: Any) -> Any:
182
+ if self.torch.cuda.current_stream(value.device).cuda_stream != entry["stream"].cuda_stream:
183
+ raise RuntimeError("Replay stream changed")
184
+ entry["static_input"].copy_(value)
185
+ entry["graph"].replay()
186
+ return entry["static_output"].clone()
187
+
188
+ def release(self, entry: dict[str, Any]) -> dict[str, Any]:
189
+ torch = self.torch
190
+ refs = {key: weakref.ref(entry[key]) for key in ("static_input", "static_output", "graph")}
191
+ events = []
192
+
193
+ def snapshot(stage: str) -> None:
194
+ item = describe_pool(torch.cuda.memory_snapshot(
195
+ mempool_id=entry["pool"], include_traces=False), entry["pool"])
196
+ item.update(stage=stage, weak_alive={key: ref() is not None for key, ref in refs.items()})
197
+ events.append(item)
198
+
199
+ entry["stream"].synchronize()
200
+ entry["side"].synchronize()
201
+ snapshot("before_drop")
202
+ entry["graph"].reset()
203
+ entry["static_input"] = entry["static_output"] = entry["graph"] = None
204
+ snapshot("after_drop")
205
+ entry["stream"].synchronize()
206
+ entry["side"].synchronize()
207
+ with torch.cuda.stream(entry["stream"]):
208
+ probe = torch.empty((1,), device=entry["stream"].device, dtype=torch.uint8)
209
+ del probe
210
+ entry["stream"].synchronize()
211
+ snapshot("after_fence_allocator_poll")
212
+ final = events[-1]
213
+ checks = {"no_graph_refs": not any(final["weak_alive"].values()),
214
+ "correct_pool": not final["foreign_pool"],
215
+ "pool_active_zero": final["active_bytes"] == 0,
216
+ "pool_allocated_zero": final["allocated_bytes"] == 0,
217
+ "no_active_blocks": final["active_allocated_blocks"] == 0,
218
+ "no_pending_blocks": final["pending_blocks"] == 0,
219
+ "known_block_states": not final["unknown_block_states"]}
220
+ return {"passed": all(checks.values()), "checks": checks, "events": events}
clairic_runtime/decoder_graph.py ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Generic startup GDN decoder graphs with owned native state prefixes."""
2
+ from __future__ import annotations
3
+
4
+ from contextlib import contextmanager
5
+ import os
6
+ import threading
7
+ import time
8
+ from types import MethodType
9
+ from typing import Any, Iterator
10
+ import weakref
11
+
12
+ import torch
13
+
14
+ from .decoder_capture import DecoderGraph, TorchBackend, callable_key
15
+ from .gdn_pool import NativeStatePool
16
+
17
+
18
+ class _SelectedCache:
19
+ """Expose one real native layer to the proved single-layer capture adapter."""
20
+
21
+ def __init__(self, cache: Any, index: int) -> None:
22
+ self.cache, self.index = cache, index
23
+ self.layers = [cache.layers[index]]
24
+
25
+ def has_previous_state(self, index: int) -> bool:
26
+ assert index == 0
27
+ return self.cache.has_previous_state(self.index)
28
+
29
+ def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor:
30
+ assert index == 0
31
+ return self.cache.update_recurrent_state(state, self.index)
32
+
33
+
34
+ class DecoderGraphs:
35
+ """One engine, CUDA stream and worker; native prefill and dense attention stay eager."""
36
+
37
+ def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None:
38
+ self.backbone = backbone
39
+ self.thread = threading.get_ident()
40
+ self.device = backbone.device
41
+ self.stream = torch.cuda.current_stream(self.device)
42
+ self.max_batch, self.buckets = max_batch, buckets
43
+ self.pool = NativeStatePool(backbone, max_batch)
44
+ self.graphs: dict[tuple[int, int], DecoderGraph] = {}
45
+ self.wrappers: list[tuple[Any, Any, Any]] = []
46
+ self.active: tuple[Any, int] | None = None
47
+ self.closed = False
48
+ self.preparing = True
49
+ self.stats = {"replay": 0, "eager": 0, "retired": 0}
50
+ self.cleanup: list[dict[str, Any]] = []
51
+ self.preparation: list[dict[str, Any]] = []
52
+ self.observe_requests = False
53
+ self.last_request: dict[str, Any] | None = None
54
+ self.active_methods: dict[int, Any] = {}
55
+
56
+ def _owner(self) -> bool:
57
+ return (not self.closed and threading.get_ident() == self.thread
58
+ and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream)
59
+
60
+ @staticmethod
61
+ def _runtime() -> tuple[Any, ...]:
62
+ import deep_gemm as dg
63
+
64
+ return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(),
65
+ tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_"))))
66
+
67
+ def prepare(self) -> None:
68
+ if not self._owner():
69
+ raise RuntimeError("Graph preparation must use the engine CUDA worker")
70
+ backend = TorchBackend(torch, self._runtime)
71
+ try:
72
+ for index in self.pool.states:
73
+ layer = self.pool.language.layers[index]
74
+ eager = layer.forward
75
+ weak_manager = weakref.ref(self)
76
+
77
+ def forward(this: Any, *args: Any, _index: int = index,
78
+ _eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any:
79
+ manager = _manager()
80
+ if manager is None:
81
+ return _eager(*args, **kwargs)
82
+ return manager._forward(_index, _eager, args, kwargs)
83
+
84
+ installed = MethodType(forward, layer)
85
+ self.wrappers.append((layer, eager, installed))
86
+ layer.forward = installed
87
+ dtype = torch.bfloat16 if index == 0 else torch.float32
88
+ for bucket in self.buckets:
89
+ began = time.perf_counter_ns()
90
+ dummy = self.pool.new_cache(bucket, previous=True)
91
+ selected = _SelectedCache(dummy, index)
92
+
93
+ def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any:
94
+ return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None)
95
+
96
+ zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(),
97
+ dtype=dtype, device=self.device)
98
+ self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero,
99
+ operation, exclusive_process=True)
100
+ entry = self.graphs[index, bucket].entry
101
+ self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype),
102
+ "capture_ms": (time.perf_counter_ns() - began) / 1e6,
103
+ "generic_zero_input": True,
104
+ "capture_mutation_restored": entry["capture_mutation_restored"],
105
+ "pool": list(entry["pool"]),
106
+ "pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"],
107
+ "static_input_storage_bytes": entry["static_input_storage_bytes"]})
108
+ self.pool.zero()
109
+ self.stream.synchronize()
110
+ self.preparing = False
111
+ except BaseException:
112
+ self.close()
113
+ raise
114
+
115
+ @contextmanager
116
+ def request(self, batch: int) -> Iterator[Any]:
117
+ if (not self._owner() or self.preparing or self.active is not None
118
+ or not 1 <= batch <= self.max_batch or torch.is_grad_enabled()
119
+ or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()):
120
+ yield None
121
+ return
122
+ cache = self.pool.new_cache(batch)
123
+ if not self.pool.owns(cache, batch):
124
+ raise RuntimeError("Fresh cache must use owned native state prefixes")
125
+ self.active = (cache, batch)
126
+ self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states}
127
+ completed = False
128
+ try:
129
+ yield cache
130
+ completed = True
131
+ finally:
132
+ try:
133
+ if self.observe_requests:
134
+ self.last_request = {"batch": batch, "completed": completed,
135
+ "cache_type": type(cache).__name__,
136
+ "native_prefixes_owned": self.pool.owns(cache, batch),
137
+ "layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape),
138
+ "conv_stride": list(cache.layers[index].conv_states.stride()),
139
+ "conv_dtype": str(cache.layers[index].conv_states.dtype),
140
+ "recurrent_shape": list(cache.layers[index].recurrent_states.shape),
141
+ "recurrent_stride": list(cache.layers[index].recurrent_states.stride()),
142
+ "recurrent_dtype": str(cache.layers[index].recurrent_states.dtype),
143
+ "has_previous_state": bool(cache.layers[index].has_previous_state)}
144
+ for index in self.pool.states]}
145
+ finally:
146
+ self.active = None
147
+ self.active_methods.clear()
148
+
149
+ @staticmethod
150
+ def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]:
151
+ layer = cache.layers[index]
152
+ return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state),
153
+ callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state),
154
+ layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size)
155
+
156
+ def diagnostics(self) -> dict[str, Any]:
157
+ """Primitive-only snapshot for separate untimed diagnostics and shutdown."""
158
+ return {"closed": self.closed, "preparing": self.preparing,
159
+ "active": self.active is not None, "owner_thread": self.thread,
160
+ "stream": self.stream.cuda_stream, "max_batch": self.max_batch,
161
+ "buckets": list(self.buckets), "graphs": len(self.graphs),
162
+ "arena_storage_bytes": self.pool.storage_bytes,
163
+ "stats": dict(self.stats), "preparation": list(self.preparation),
164
+ "observe_requests": self.observe_requests,
165
+ "last_request": self.last_request, "cleanup": list(self.cleanup)}
166
+
167
+ def _retire(self, index: int) -> None:
168
+ for key in tuple(self.graphs):
169
+ if key[0] == index:
170
+ graph = self.graphs[key]
171
+ details = graph.close()
172
+ del self.graphs[key]
173
+ self.cleanup.append({"layer": key[0], "bucket": key[1], **details})
174
+ if not details["passed"]:
175
+ raise RuntimeError("Decoder graph retirement failed strict cleanup")
176
+ self.stats["retired"] += 1
177
+
178
+ def _owns_layer(self, cache: Any, batch: int, index: int) -> bool:
179
+ layer = cache.layers[index]
180
+ for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True):
181
+ if (actual is None or actual.data_ptr() != owned.data_ptr()
182
+ or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
183
+ or actual.dtype != owned.dtype or actual.device != owned.device
184
+ or actual.storage_offset() != owned.storage_offset()
185
+ or tuple(actual.shape) != (batch, *owned.shape[1:])
186
+ or tuple(actual.stride()) != tuple(owned.stride())):
187
+ return False
188
+ return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized
189
+
190
+ def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any:
191
+ value = args[0] if args else kwargs.get("hidden_states")
192
+ cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None)
193
+ mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None)
194
+ batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0
195
+ expected_dtype = torch.bfloat16 if index == 0 else torch.float32
196
+ admitted = (not self.preparing and self._owner() and self.active is not None
197
+ and cache is self.active[0] and batch == self.active[1]
198
+ and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype
199
+ and value.device == self.device and mask is None and not torch.is_grad_enabled()
200
+ and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
201
+ and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids",
202
+ "past_key_values", "use_cache", "cache_position"}
203
+ and self._cache_methods(cache, index) == self.active_methods.get(index)
204
+ and self._owns_layer(cache, batch, index) and cache.has_previous_state(index))
205
+ if not admitted:
206
+ self.stats["eager"] += 1
207
+ return eager(*args, **kwargs)
208
+ bucket = next((bound for bound in self.buckets if batch <= bound), None)
209
+ graph = self.graphs.get((index, bucket))
210
+ if graph is None:
211
+ self.stats["eager"] += 1
212
+ return eager(*args, **kwargs)
213
+ entry = graph.entry
214
+ try:
215
+ graph.check(entry["static_input"])
216
+ except RuntimeError:
217
+ self._retire(index)
218
+ self.stats["eager"] += 1
219
+ return eager(*args, **kwargs)
220
+ entry["static_input"][:batch].copy_(value)
221
+ if batch < bucket:
222
+ entry["static_input"][batch:].zero_()
223
+ entry["graph"].replay()
224
+ self.stats["replay"] += 1
225
+ return entry["static_output"][:batch].clone()
226
+
227
+ def close(self) -> None:
228
+ if self.closed:
229
+ return
230
+ if not self._owner():
231
+ raise RuntimeError("Graph cleanup must use the engine CUDA worker")
232
+ self.active = None
233
+ self.active_methods.clear()
234
+ error = None
235
+ try:
236
+ for index in tuple(self.pool.states):
237
+ try:
238
+ self._retire(index)
239
+ except BaseException as exc:
240
+ if error is None:
241
+ error = exc
242
+ finally:
243
+ for layer, original, installed in reversed(self.wrappers):
244
+ if layer.forward is installed:
245
+ layer.forward = original
246
+ self.wrappers.clear()
247
+ self.pool.close()
248
+ self.backbone = None
249
+ self.closed = True
250
+ if error is not None:
251
+ raise error
252
+
253
+
254
+ def _admissible(backbone: Any, max_batch: int) -> bool:
255
+ """Reject unsupported contexts before allocating arenas or installing wrappers."""
256
+ config = backbone.config.get_text_config()
257
+ limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0
258
+ if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16
259
+ or not 1 <= max_batch <= limit):
260
+ return False
261
+ registry = torch.nn.modules.module
262
+ if registry._global_forward_hooks or registry._global_forward_pre_hooks:
263
+ return False
264
+ for module in backbone.modules():
265
+ if module._forward_hooks or module._forward_pre_hooks:
266
+ return False
267
+ for layer in backbone.model.language_model.layers:
268
+ if layer.layer_type != "linear_attention":
269
+ continue
270
+ for value in layer.parameters():
271
+ try:
272
+ version = value._version
273
+ except RuntimeError:
274
+ return False
275
+ if version is None:
276
+ return False
277
+ return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
278
+
279
+
280
+ def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None:
281
+ """Run on the existing worker after math installation; unsupported engines stay eager."""
282
+ if not _admissible(backbone, max_batch):
283
+ return None
284
+ largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound)
285
+ buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest)
286
+ with torch.no_grad():
287
+ manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets)
288
+ manager.prepare()
289
+ return manager
clairic_runtime/gdn/LICENSES ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Flash Linear Attention — MIT
2
+ ============================
3
+
4
+ MIT License
5
+
6
+ Copyright (c) 2023-2025 Songlin Yang
7
+
8
+ Permission is hereby granted, free of charge, to any person obtaining a copy
9
+ of this software and associated documentation files (the "Software"), to deal
10
+ in the Software without restriction, including without limitation the rights
11
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12
+ copies of the Software, and to permit persons to whom the Software is
13
+ furnished to do so, subject to the following conditions:
14
+
15
+ The above copyright notice and this permission notice shall be included in all
16
+ copies or substantial portions of the Software.
17
+
18
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24
+ SOFTWARE.
25
+
26
+ SGLang — Apache-2.0
27
+ ===================
28
+
29
+ Apache License
30
+ Version 2.0, January 2004
31
+ http://www.apache.org/licenses/
32
+
33
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
34
+
35
+ 1. Definitions.
36
+
37
+ "License" shall mean the terms and conditions for use, reproduction,
38
+ and distribution as defined by Sections 1 through 9 of this document.
39
+
40
+ "Licensor" shall mean the copyright owner or entity authorized by
41
+ the copyright owner that is granting the License.
42
+
43
+ "Legal Entity" shall mean the union of the acting entity and all
44
+ other entities that control, are controlled by, or are under common
45
+ control with that entity. For the purposes of this definition,
46
+ "control" means (i) the power, direct or indirect, to cause the
47
+ direction or management of such entity, whether by contract or
48
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
49
+ outstanding shares, or (iii) beneficial ownership of such entity.
50
+
51
+ "You" (or "Your") shall mean an individual or Legal Entity
52
+ exercising permissions granted by this License.
53
+
54
+ "Source" form shall mean the preferred form for making modifications,
55
+ including but not limited to software source code, documentation
56
+ source, and configuration files.
57
+
58
+ "Object" form shall mean any form resulting from mechanical
59
+ transformation or translation of a Source form, including but
60
+ not limited to compiled object code, generated documentation,
61
+ and conversions to other media types.
62
+
63
+ "Work" shall mean the work of authorship, whether in Source or
64
+ Object form, made available under the License, as indicated by a
65
+ copyright notice that is included in or attached to the work
66
+ (an example is provided in the Appendix below).
67
+
68
+ "Derivative Works" shall mean any work, whether in Source or Object
69
+ form, that is based on (or derived from) the Work and for which the
70
+ editorial revisions, annotations, elaborations, or other modifications
71
+ represent, as a whole, an original work of authorship. For the purposes
72
+ of this License, Derivative Works shall not include works that remain
73
+ separable from, or merely link (or bind by name) to the interfaces of,
74
+ the Work and Derivative Works thereof.
75
+
76
+ "Contribution" shall mean any work of authorship, including
77
+ the original version of the Work and any modifications or additions
78
+ to that Work or Derivative Works thereof, that is intentionally
79
+ submitted to Licensor for inclusion in the Work by the copyright owner
80
+ or by an individual or Legal Entity authorized to submit on behalf of
81
+ the copyright owner. For the purposes of this definition, "submitted"
82
+ means any form of electronic, verbal, or written communication sent
83
+ to the Licensor or its representatives, including but not limited to
84
+ communication on electronic mailing lists, source code control systems,
85
+ and issue tracking systems that are managed by, or on behalf of, the
86
+ Licensor for the purpose of discussing and improving the Work, but
87
+ excluding communication that is conspicuously marked or otherwise
88
+ designated in writing by the copyright owner as "Not a Contribution."
89
+
90
+ "Contributor" shall mean Licensor and any individual or Legal Entity
91
+ on behalf of whom a Contribution has been received by Licensor and
92
+ subsequently incorporated within the Work.
93
+
94
+ 2. Grant of Copyright License. Subject to the terms and conditions of
95
+ this License, each Contributor hereby grants to You a perpetual,
96
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
97
+ copyright license to reproduce, prepare Derivative Works of,
98
+ publicly display, publicly perform, sublicense, and distribute the
99
+ Work and such Derivative Works in Source or Object form.
100
+
101
+ 3. Grant of Patent License. Subject to the terms and conditions of
102
+ this License, each Contributor hereby grants to You a perpetual,
103
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
104
+ (except as stated in this section) patent license to make, have made,
105
+ use, offer to sell, sell, import, and otherwise transfer the Work,
106
+ where such license applies only to those patent claims licensable
107
+ by such Contributor that are necessarily infringed by their
108
+ Contribution(s) alone or by combination of their Contribution(s)
109
+ with the Work to which such Contribution(s) was submitted. If You
110
+ institute patent litigation against any entity (including a
111
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
112
+ or a Contribution incorporated within the Work constitutes direct
113
+ or contributory patent infringement, then any patent licenses
114
+ granted to You under this License for that Work shall terminate
115
+ as of the date such litigation is filed.
116
+
117
+ 4. Redistribution. You may reproduce and distribute copies of the
118
+ Work or Derivative Works thereof in any medium, with or without
119
+ modifications, and in Source or Object form, provided that You
120
+ meet the following conditions:
121
+
122
+ (a) You must give any other recipients of the Work or
123
+ Derivative Works a copy of this License; and
124
+
125
+ (b) You must cause any modified files to carry prominent notices
126
+ stating that You changed the files; and
127
+
128
+ (c) You must retain, in the Source form of any Derivative Works
129
+ that You distribute, all copyright, patent, trademark, and
130
+ attribution notices from the Source form of the Work,
131
+ excluding those notices that do not pertain to any part of
132
+ the Derivative Works; and
133
+
134
+ (d) If the Work includes a "NOTICE" text file as part of its
135
+ distribution, then any Derivative Works that You distribute must
136
+ include a readable copy of the attribution notices contained
137
+ within such NOTICE file, excluding those notices that do not
138
+ pertain to any part of the Derivative Works, in at least one
139
+ of the following places: within a NOTICE text file distributed
140
+ as part of the Derivative Works; within the Source form or
141
+ documentation, if provided along with the Derivative Works; or,
142
+ within a display generated by the Derivative Works, if and
143
+ wherever such third-party notices normally appear. The contents
144
+ of the NOTICE file are for informational purposes only and
145
+ do not modify the License. You may add Your own attribution
146
+ notices within Derivative Works that You distribute, alongside
147
+ or as an addendum to the NOTICE text from the Work, provided
148
+ that such additional attribution notices cannot be construed
149
+ as modifying the License.
150
+
151
+ You may add Your own copyright statement to Your modifications and
152
+ may provide additional or different license terms and conditions
153
+ for use, reproduction, or distribution of Your modifications, or
154
+ for any such Derivative Works as a whole, provided Your use,
155
+ reproduction, and distribution of the Work otherwise complies with
156
+ the conditions stated in this License.
157
+
158
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
159
+ any Contribution intentionally submitted for inclusion in the Work
160
+ by You to the Licensor shall be under the terms and conditions of
161
+ this License, without any additional terms or conditions.
162
+ Notwithstanding the above, nothing herein shall supersede or modify
163
+ the terms of any separate license agreement you may have executed
164
+ with Licensor regarding such Contributions.
165
+
166
+ 6. Trademarks. This License does not grant permission to use the trade
167
+ names, trademarks, service marks, or product names of the Licensor,
168
+ except as required for reasonable and customary use in describing the
169
+ origin of the Work and reproducing the content of the NOTICE file.
170
+
171
+ 7. Disclaimer of Warranty. Unless required by applicable law or
172
+ agreed to in writing, Licensor provides the Work (and each
173
+ Contributor provides its Contributions) on an "AS IS" BASIS,
174
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
175
+ implied, including, without limitation, any warranties or conditions
176
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
177
+ PARTICULAR PURPOSE. You are solely responsible for determining the
178
+ appropriateness of using or redistributing the Work and assume any
179
+ risks associated with Your exercise of permissions under this License.
180
+
181
+ 8. Limitation of Liability. In no event and under no legal theory,
182
+ whether in tort (including negligence), contract, or otherwise,
183
+ unless required by applicable law (such as deliberate and grossly
184
+ negligent acts) or agreed to in writing, shall any Contributor be
185
+ liable to You for damages, including any direct, indirect, special,
186
+ incidental, or consequential damages of any character arising as a
187
+ result of this License or out of the use or inability to use the
188
+ Work (including but not limited to damages for loss of goodwill,
189
+ work stoppage, computer failure or malfunction, or any and all
190
+ other commercial damages or losses), even if such Contributor
191
+ has been advised of the possibility of such damages.
192
+
193
+ 9. Accepting Warranty or Additional Liability. While redistributing
194
+ the Work or Derivative Works thereof, You may choose to offer,
195
+ and charge a fee for, acceptance of support, warranty, indemnity,
196
+ or other liability obligations and/or rights consistent with this
197
+ License. However, in accepting such obligations, You may act only
198
+ on Your own behalf and on Your sole responsibility, not on behalf
199
+ of any other Contributor, and only if You agree to indemnify,
200
+ defend, and hold each Contributor harmless for any liability
201
+ incurred by, or claims asserted against, such Contributor by reason
202
+ of your accepting any such warranty or additional liability.
203
+
204
+ END OF TERMS AND CONDITIONS
205
+
206
+ APPENDIX: How to apply the Apache License to your work.
207
+
208
+ To apply the Apache License to your work, attach the following
209
+ boilerplate notice, with the fields enclosed by brackets "[]"
210
+ replaced with your own identifying information. (Don't include
211
+ the brackets!) The text should be enclosed in the appropriate
212
+ comment syntax for the file format. We also recommend that a
213
+ file or class name and description of purpose be included on the
214
+ same "printed page" as the copyright notice for easier
215
+ identification within third-party archives.
216
+
217
+ Copyright 2023-2024 SGLang Team
218
+
219
+ Licensed under the Apache License, Version 2.0 (the "License");
220
+ you may not use this file except in compliance with the License.
221
+ You may obtain a copy of the License at
222
+
223
+ http://www.apache.org/licenses/LICENSE-2.0
224
+
225
+ Unless required by applicable law or agreed to in writing, software
226
+ distributed under the License is distributed on an "AS IS" BASIS,
227
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
228
+ See the License for the specific language governing permissions and
229
+ limitations under the License.
clairic_runtime/gdn/NOTICE ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Standalone GDN inference helpers
2
+
3
+ These files contain portions of Flash Linear Attention (FLA),
4
+ https://github.com/fla-org/flash-linear-attention, under the MIT License.
5
+ Copyright (c) 2023-2025 Songlin Yang.
6
+ The retained source headers also credit Songlin Yang and Yu Zhang.
7
+
8
+ They include adaptations from SGLang,
9
+ https://github.com/sgl-project/sglang, under the Apache License, Version 2.0.
10
+ Applicable original source headers are retained.
11
+
12
+ The module import layout and standalone helper plumbing were adapted.
13
+ The selected inference kernel bodies and launch configurations are retained.
14
+ Full license texts are provided in LICENSES.
clairic_runtime/gdn/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Gated delta network inference operations."""
clairic_runtime/gdn/adapter.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pack native inputs and preserve native cache axes around selected kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ import torch
8
+
9
+ from .. import recurrent_metadata
10
+
11
+
12
+ def _packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
13
+ """Return valid-token indices and one boundary per native row."""
14
+ batch, length = mixed.shape[:2]
15
+ if mask is None:
16
+ indices = torch.arange(batch * length, device=mixed.device)
17
+ boundaries = torch.arange(batch + 1, device=mixed.device, dtype=torch.int32) * length
18
+ return indices, boundaries
19
+ if mask.ndim != 2 or mask.shape[0] != batch or mask.shape[1] < length:
20
+ raise ValueError("Expected a two-dimensional mask covering current tokens")
21
+ valid = mask[:, -length:].to(device=mixed.device, dtype=torch.bool)
22
+ lengths = valid.sum(dim=1, dtype=torch.int32)
23
+ if bool((lengths == 0).any()):
24
+ raise ValueError("Each recurrent sequence must contain a current token")
25
+ boundaries = torch.cat((lengths.new_zeros(1), lengths.cumsum(0, dtype=torch.int32)))
26
+ return valid.reshape(-1).nonzero().flatten(), boundaries
27
+
28
+
29
+ def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
30
+ """Reuse exact packing metadata within an owned inference forward."""
31
+ return recurrent_metadata.packed_indices(mixed, mask, _packed_indices)
32
+
33
+
34
+ def source_state(native: torch.Tensor | None, batch: int, heads: int, key_dim: int,
35
+ value_dim: int, reference: torch.Tensor) -> torch.Tensor:
36
+ """Map native K,V cache dimensions to the selected kernel's V,K layout."""
37
+ if native is None:
38
+ return torch.zeros(batch, heads, value_dim, key_dim,
39
+ device=reference.device, dtype=torch.float32)
40
+ if tuple(native.shape) != (batch, heads, key_dim, value_dim):
41
+ raise ValueError("Unexpected native recurrent cache shape")
42
+ if native.dtype != torch.float32:
43
+ raise ValueError("Recurrent states must retain float32 precision")
44
+ return native.transpose(-1, -2).contiguous()
45
+
46
+
47
+ def native_state(source: torch.Tensor) -> torch.Tensor:
48
+ """Expose K,V cache axes without narrowing or copying source state."""
49
+ return source.transpose(-1, -2)
50
+
51
+
52
+ def store_state(cache: Any, layer_idx: int, state: torch.Tensor) -> torch.Tensor:
53
+ """Initialize FP32 recurrent storage while retaining native cache lifecycle."""
54
+ if state.dtype != torch.float32:
55
+ raise ValueError("Expected float32 recurrent state")
56
+ layer = cache.layers[layer_idx]
57
+ if not layer.is_recurrent_states_initialized:
58
+ layer.recurrent_states = torch.empty_like(state)
59
+ if not torch.compiler.is_compiling():
60
+ torch._dynamo.mark_static_address(layer.recurrent_states)
61
+ layer.is_recurrent_states_initialized = True
62
+ elif layer.recurrent_states.dtype != torch.float32:
63
+ raise ValueError("Cannot resume a narrowed recurrent cache")
64
+ return cache.update_recurrent_state(state, layer_idx)
65
+
66
+
67
+ def prefill(mixed: torch.Tensor, a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor,
68
+ dt_bias: torch.Tensor, key_heads: int, value_heads: int, key_dim: int,
69
+ value_dim: int, initial_state: torch.Tensor | None,
70
+ attention_mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
71
+ """Run packed prefill and scatter valid outputs to native padded shape."""
72
+ from .gating import fused_gdn_gating
73
+ from .index import prepare_chunk_indices
74
+ from .l2norm import l2norm_fwd
75
+ from .prefill import chunk_gated_delta_rule_fwd
76
+
77
+ batch, length, _ = mixed.shape
78
+ indices, boundaries = packed_indices(mixed, attention_mask)
79
+ packed = mixed.flatten(0, 1).index_select(0, indices)
80
+ pa = a.flatten(0, 1).index_select(0, indices)
81
+ pb = b.flatten(0, 1).index_select(0, indices)
82
+ query, key, value = torch.split(packed,
83
+ (key_heads * key_dim, key_heads * key_dim, value_heads * value_dim), dim=-1)
84
+ query = l2norm_fwd(query.reshape(1, -1, key_heads, key_dim).contiguous())
85
+ key = l2norm_fwd(key.reshape(1, -1, key_heads, key_dim).contiguous())
86
+ value = value.reshape(1, -1, value_heads, value_dim).contiguous()
87
+ gate, beta = fused_gdn_gating(A_log, pa, pb, dt_bias)
88
+ state = source_state(initial_state, batch, value_heads, key_dim, value_dim, mixed)
89
+ slots = recurrent_metadata.sequence_slots(mixed)
90
+ _, output, *_ = chunk_gated_delta_rule_fwd(query, key, value, gate, beta,
91
+ key_dim ** -0.5, state, slots, boundaries, prepare_chunk_indices(boundaries, 64))
92
+ padded = mixed.new_zeros(batch * length, value_heads, value_dim)
93
+ padded.index_copy_(0, indices, output.squeeze(0))
94
+ return padded.reshape(batch, length, value_heads, value_dim), native_state(state)
95
+
96
+
97
+ def decode(mixed: torch.Tensor, a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor,
98
+ dt_bias: torch.Tensor, initial_state: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
99
+ """Use the selected packed single-token recurrent operation."""
100
+ from .recurrent import fused_recurrent_gated_delta_rule_packed_decode
101
+
102
+ batch, length, _ = mixed.shape
103
+ if length != 1:
104
+ raise ValueError("Packed recurrent decode requires one token per row")
105
+ _, heads, key_dim, value_dim = initial_state.shape
106
+ state = source_state(initial_state, batch, heads, key_dim, value_dim, mixed)
107
+ output = mixed.new_empty(batch, 1, heads, value_dim)
108
+ slots = recurrent_metadata.sequence_slots(mixed)
109
+ fused_recurrent_gated_delta_rule_packed_decode(mixed[:, 0].contiguous(),
110
+ a[:, 0].contiguous(), b[:, 0].contiguous(), A_log, dt_bias, key_dim ** -0.5,
111
+ state, output, slots, use_qk_l2norm_in_kernel=True)
112
+ return output, native_state(state)
clairic_runtime/gdn/chunk_delta_h.py ADDED
@@ -0,0 +1,386 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+ # Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_delta_h.py
3
+ # -*- coding: utf-8 -*-
4
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
5
+
6
+ from typing import Optional, Tuple
7
+
8
+ import torch
9
+ import triton
10
+ import triton.language as tl
11
+
12
+ from .index import (
13
+ prepare_chunk_indices,
14
+ prepare_chunk_offsets,
15
+ )
16
+ from .op import exp, exp2, safe_exp
17
+ from .utils import (
18
+ autotune_cache_kwargs,
19
+ is_nvidia_hopper,
20
+ )
21
+
22
+ NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16]
23
+ CHUNK_SIZE = 64
24
+ GDN_CHUNK_H_BV = 32
25
+ GDN_CHUNK_H_NUM_WARPS = 4
26
+ GDN_CHUNK_H_NUM_STAGES = 2
27
+
28
+
29
+ @triton.autotune(
30
+ # Single hardcoded config. The kernel writes ht (final state) back into
31
+ # initial_state in-place; with multiple configs, triton's autotuning
32
+ # phase invokes the kernel many times for timing and corrupts the cache pool,
33
+ # producing silently wrong output on the first user request. Restoring via
34
+ # `restore_value=["initial_state"]` works for unit tests but OOMs on
35
+ # production-scale models (e.g. Kimi-Linear-48B at default mem_fraction)
36
+ # because cloning the cache pool for each configuration exceeds available memory.
37
+ # NT_BUCKET is kept in the autotune key for forward-compatibility (allows
38
+ # future per-bucket configs once the kernel is refactored to write final
39
+ # state to a separate output buffer). The env knobs keep this single-config
40
+ # property while allowing model/hardware-local validation of the selected
41
+ # tile without corrupting the state pool through multi-config autotune.
42
+ configs=[
43
+ triton.Config(
44
+ {"BV": GDN_CHUNK_H_BV},
45
+ num_warps=GDN_CHUNK_H_NUM_WARPS,
46
+ num_stages=GDN_CHUNK_H_NUM_STAGES,
47
+ )
48
+ ],
49
+ key=["H", "K", "V", "BT", "USE_GK", "NT_BUCKET"],
50
+ **autotune_cache_kwargs,
51
+ )
52
+ @triton.jit(do_not_specialize=["T"])
53
+ def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
54
+ k: tl.tensor,
55
+ v: tl.tensor,
56
+ w: tl.tensor,
57
+ v_new: tl.tensor,
58
+ g: tl.tensor,
59
+ gk: tl.tensor,
60
+ h: tl.tensor,
61
+ initial_state: tl.tensor,
62
+ initial_state_indices: tl.tensor,
63
+ stride_init_state: tl.tensor,
64
+ cu_seqlens: tl.tensor,
65
+ chunk_offsets: tl.tensor,
66
+ T: tl.int32,
67
+ H: tl.constexpr,
68
+ Hg: tl.constexpr,
69
+ K: tl.constexpr,
70
+ V: tl.constexpr,
71
+ BT: tl.constexpr,
72
+ BV: tl.constexpr,
73
+ USE_G: tl.constexpr,
74
+ USE_GK: tl.constexpr,
75
+ USE_INITIAL_STATE: tl.constexpr,
76
+ INPLACE_UPDATE: tl.constexpr,
77
+ SAVE_NEW_VALUE: tl.constexpr,
78
+ IS_VARLEN: tl.constexpr,
79
+ NT_BUCKET: tl.constexpr,
80
+ USE_EXP2: tl.constexpr,
81
+ ) -> None:
82
+ i_v, i_nh = tl.program_id(0), tl.program_id(1)
83
+ i_n, i_h = i_nh // H, i_nh % H
84
+ if IS_VARLEN:
85
+ bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
86
+ cu_seqlens + i_n + 1
87
+ ).to(tl.int32)
88
+ T = eos - bos
89
+ NT = tl.cdiv(T, BT)
90
+ boh = tl.load(chunk_offsets + i_n).to(tl.int32)
91
+ else:
92
+ bos, eos = i_n * T, i_n * T + T
93
+ NT = tl.cdiv(T, BT)
94
+ boh = i_n * NT
95
+
96
+ # [BV, BK]
97
+ b_h1 = tl.zeros([BV, 64], dtype=tl.float32)
98
+ if K > 64:
99
+ b_h2 = tl.zeros([BV, 64], dtype=tl.float32)
100
+ if K > 128:
101
+ b_h3 = tl.zeros([BV, 64], dtype=tl.float32)
102
+ if K > 192:
103
+ b_h4 = tl.zeros([BV, 64], dtype=tl.float32)
104
+
105
+ # calculate offset
106
+ h += ((boh * H + i_h) * V * K).to(tl.int64)
107
+ v += ((bos * H + i_h) * V).to(tl.int64)
108
+ k += ((bos * Hg + i_h // (H // Hg)) * K).to(tl.int64)
109
+ w += ((bos * H + i_h) * K).to(tl.int64)
110
+ if SAVE_NEW_VALUE:
111
+ v_new += ((bos * H + i_h) * V).to(tl.int64)
112
+ stride_v = H * V
113
+ stride_h = H * V * K
114
+ stride_k = Hg * K
115
+ stride_w = H * K
116
+
117
+ # Slot stride comes from the caller (initial_state.stride(0)): the state pool
118
+ # may be an envelope-strided view (page-major / unified memory), where the
119
+ # per-slot pitch spans ALL layers' state, not H*V*K. int64: envelope pitches
120
+ # overflow an int32 index product.
121
+ index = tl.load(initial_state_indices + i_n).to(tl.int64)
122
+ # Padded rows carry the -1 sentinel; the decode kernel guards on it
123
+ # (fused_recurrent.py), the chunked extend path did not.
124
+ valid_state = index >= 0
125
+ h0 = initial_state + index * stride_init_state
126
+ ht = initial_state + index * stride_init_state
127
+ if USE_INITIAL_STATE:
128
+ h0 = h0 + i_h * V * K
129
+ if INPLACE_UPDATE:
130
+ ht = ht + i_h * V * K
131
+
132
+ # load initial state
133
+ if USE_INITIAL_STATE and valid_state:
134
+ p_h0_1 = tl.make_block_ptr(h0, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
135
+ b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32)
136
+ if K > 64:
137
+ p_h0_2 = tl.make_block_ptr(
138
+ h0, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
139
+ )
140
+ b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32)
141
+ if K > 128:
142
+ p_h0_3 = tl.make_block_ptr(
143
+ h0, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
144
+ )
145
+ b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32)
146
+ if K > 192:
147
+ p_h0_4 = tl.make_block_ptr(
148
+ h0, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
149
+ )
150
+ b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32)
151
+
152
+ # main recurrence
153
+ for i_t in range(NT):
154
+ p_h1 = tl.make_block_ptr(
155
+ h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0)
156
+ )
157
+ tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1))
158
+ if K > 64:
159
+ p_h2 = tl.make_block_ptr(
160
+ h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
161
+ )
162
+ tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1))
163
+ if K > 128:
164
+ p_h3 = tl.make_block_ptr(
165
+ h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
166
+ )
167
+ tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1))
168
+ if K > 192:
169
+ p_h4 = tl.make_block_ptr(
170
+ h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
171
+ )
172
+ tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1))
173
+
174
+ p_w = tl.make_block_ptr(
175
+ w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0)
176
+ )
177
+ b_w = tl.load(p_w, boundary_check=(0, 1))
178
+ b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype))
179
+ if K > 64:
180
+ p_w = tl.make_block_ptr(
181
+ w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0)
182
+ )
183
+ b_w = tl.load(p_w, boundary_check=(0, 1))
184
+ b_v += tl.dot(b_w, tl.trans(b_h2).to(b_w.dtype))
185
+ if K > 128:
186
+ p_w = tl.make_block_ptr(
187
+ w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0)
188
+ )
189
+ b_w = tl.load(p_w, boundary_check=(0, 1))
190
+ b_v += tl.dot(b_w, tl.trans(b_h3).to(b_w.dtype))
191
+ if K > 192:
192
+ p_w = tl.make_block_ptr(
193
+ w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0)
194
+ )
195
+ b_w = tl.load(p_w, boundary_check=(0, 1))
196
+ b_v += tl.dot(b_w, tl.trans(b_h4).to(b_w.dtype))
197
+ p_v = tl.make_block_ptr(
198
+ v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
199
+ )
200
+ b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v
201
+
202
+ if SAVE_NEW_VALUE:
203
+ p_v = tl.make_block_ptr(
204
+ v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
205
+ )
206
+ tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1))
207
+
208
+ last_idx = min((i_t + 1) * BT, T) - 1
209
+ if USE_G:
210
+ b_g_last = tl.load(g + bos * H + last_idx * H + i_h)
211
+ p_g = tl.make_block_ptr(
212
+ g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
213
+ )
214
+ b_g = tl.load(p_g, boundary_check=(0,))
215
+ b_v = b_v * safe_exp(b_g_last - b_g)[:, None]
216
+ b_g_last = exp(b_g_last)
217
+ b_h1 = b_h1 * b_g_last
218
+ if K > 64:
219
+ b_h2 = b_h2 * b_g_last
220
+ if K > 128:
221
+ b_h3 = b_h3 * b_g_last
222
+ if K > 192:
223
+ b_h4 = b_h4 * b_g_last
224
+
225
+ if USE_GK:
226
+ o_k1 = tl.arange(0, 64)
227
+ b_gk_last1 = tl.load(
228
+ gk + (bos + last_idx) * H * K + i_h * K + o_k1,
229
+ mask=(o_k1 < K),
230
+ other=0.0,
231
+ )
232
+ if USE_EXP2:
233
+ b_h1 *= exp2(b_gk_last1)[None, :]
234
+ else:
235
+ b_h1 *= exp(b_gk_last1)[None, :]
236
+ if K > 64:
237
+ o_k2 = 64 + o_k1
238
+ b_gk_last2 = tl.load(
239
+ gk + (bos + last_idx) * H * K + i_h * K + o_k2,
240
+ mask=(o_k2 < K),
241
+ other=0.0,
242
+ )
243
+ if USE_EXP2:
244
+ b_h2 *= exp2(b_gk_last2)[None, :]
245
+ else:
246
+ b_h2 *= exp(b_gk_last2)[None, :]
247
+ if K > 128:
248
+ o_k3 = 128 + o_k1
249
+ b_gk_last3 = tl.load(
250
+ gk + (bos + last_idx) * H * K + i_h * K + o_k3,
251
+ mask=(o_k3 < K),
252
+ other=0.0,
253
+ )
254
+ if USE_EXP2:
255
+ b_h3 *= exp2(b_gk_last3)[None, :]
256
+ else:
257
+ b_h3 *= exp(b_gk_last3)[None, :]
258
+ if K > 192:
259
+ o_k4 = 192 + o_k1
260
+ b_gk_last4 = tl.load(
261
+ gk + (bos + last_idx) * H * K + i_h * K + o_k4,
262
+ mask=(o_k4 < K),
263
+ other=0.0,
264
+ )
265
+ if USE_EXP2:
266
+ b_h4 *= exp2(b_gk_last4)[None, :]
267
+ else:
268
+ b_h4 *= exp(b_gk_last4)[None, :]
269
+ b_v = b_v.to(k.dtype.element_ty)
270
+
271
+ p_k = tl.make_block_ptr(
272
+ k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)
273
+ )
274
+ b_k = tl.load(p_k, boundary_check=(0, 1))
275
+ b_h1 += tl.trans(tl.dot(b_k, b_v))
276
+ if K > 64:
277
+ p_k = tl.make_block_ptr(
278
+ k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)
279
+ )
280
+ b_k = tl.load(p_k, boundary_check=(0, 1))
281
+ b_h2 += tl.trans(tl.dot(b_k, b_v))
282
+ if K > 128:
283
+ p_k = tl.make_block_ptr(
284
+ k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)
285
+ )
286
+ b_k = tl.load(p_k, boundary_check=(0, 1))
287
+ b_h3 += tl.trans(tl.dot(b_k, b_v))
288
+ if K > 192:
289
+ p_k = tl.make_block_ptr(
290
+ k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)
291
+ )
292
+ b_k = tl.load(p_k, boundary_check=(0, 1))
293
+ b_h4 += tl.trans(tl.dot(b_k, b_v))
294
+
295
+ # epilogue
296
+ if INPLACE_UPDATE and valid_state:
297
+ p_ht = tl.make_block_ptr(ht, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
298
+ tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
299
+ if K > 64:
300
+ p_ht = tl.make_block_ptr(
301
+ ht, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
302
+ )
303
+ tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
304
+ if K > 128:
305
+ p_ht = tl.make_block_ptr(
306
+ ht, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
307
+ )
308
+ tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
309
+ if K > 192:
310
+ p_ht = tl.make_block_ptr(
311
+ ht, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
312
+ )
313
+ tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
314
+
315
+
316
+ def chunk_gated_delta_rule_fwd_h(
317
+ k: torch.Tensor,
318
+ w: torch.Tensor,
319
+ u: torch.Tensor,
320
+ g: Optional[torch.Tensor] = None,
321
+ gk: Optional[torch.Tensor] = None,
322
+ initial_state: Optional[torch.Tensor] = None,
323
+ initial_state_indices: Optional[torch.Tensor] = None,
324
+ save_new_value: bool = True,
325
+ cu_seqlens: Optional[torch.LongTensor] = None,
326
+ chunk_indices: Optional[torch.LongTensor] = None,
327
+ use_exp2: bool = False,
328
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
329
+ assert not (
330
+ use_exp2 and g is not None
331
+ ), "use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
332
+ B, T, Hg, K, V = *k.shape, u.shape[-1]
333
+ H = u.shape[-2]
334
+ BT = CHUNK_SIZE
335
+
336
+ if chunk_indices is None and cu_seqlens is not None:
337
+ chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
338
+ # N: the actual number of sequences in the batch with either equal or variable lengths
339
+ if cu_seqlens is None:
340
+ N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
341
+ else:
342
+ N, NT, chunk_offsets = (
343
+ len(cu_seqlens) - 1,
344
+ len(chunk_indices),
345
+ prepare_chunk_offsets(cu_seqlens, BT),
346
+ )
347
+ assert K <= 256, "current kernel does not support head dimension larger than 256."
348
+
349
+ h = k.new_empty(B, NT, H, V, K)
350
+
351
+ v_new = torch.empty_like(u) if save_new_value else None
352
+
353
+ def grid(meta: Any) -> Any:
354
+ return (triton.cdiv(V, meta["BV"]), N * H)
355
+
356
+ chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid](
357
+ k=k,
358
+ v=u,
359
+ w=w,
360
+ v_new=v_new,
361
+ g=g,
362
+ gk=gk,
363
+ h=h,
364
+ initial_state=initial_state,
365
+ initial_state_indices=initial_state_indices,
366
+ # Envelope-strided state pools (page-major / unified memory) have a
367
+ # per-slot pitch != H*V*K; contiguous pools pass exactly H*V*K.
368
+ stride_init_state=(initial_state.stride(0) if initial_state is not None else 0),
369
+ cu_seqlens=cu_seqlens,
370
+ chunk_offsets=chunk_offsets,
371
+ T=T,
372
+ H=H,
373
+ Hg=Hg,
374
+ K=K,
375
+ V=V,
376
+ BT=BT,
377
+ USE_G=g is not None,
378
+ USE_GK=gk is not None,
379
+ USE_INITIAL_STATE=initial_state is not None,
380
+ INPLACE_UPDATE=True,
381
+ SAVE_NEW_VALUE=v_new is not None,
382
+ IS_VARLEN=cu_seqlens is not None,
383
+ NT_BUCKET=(0 if NT <= 32 else (1 if NT <= 128 else 2)),
384
+ USE_EXP2=use_exp2,
385
+ )
386
+ return h, v_new
clairic_runtime/gdn/chunk_fwd.py ADDED
@@ -0,0 +1,417 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/chunk_fwd.py
2
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
3
+
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .index import prepare_chunk_indices
10
+ from .op import safe_exp
11
+ from .utils import (
12
+ autotune_cache_kwargs,
13
+ is_tf32_supported,
14
+ )
15
+ from .wy_fast import recompute_w_u_fwd
16
+
17
+ # TF32 for the block-merge dot products (16x16 matmuls) is safe and ~2x faster on SM90.
18
+ # The numerically sensitive forward-substitution uses scalar ops, not tl.dot.
19
+ if is_tf32_supported:
20
+ _MERGE_DOT_PRECISION = tl.constexpr("tf32")
21
+ else:
22
+ _MERGE_DOT_PRECISION = tl.constexpr("ieee")
23
+
24
+
25
+ @triton.heuristics(
26
+ {
27
+ "USE_G": lambda args: args["g"] is not None,
28
+ "IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
29
+ }
30
+ )
31
+ @triton.autotune(
32
+ configs=[
33
+ triton.Config({"BK": BK}, num_warps=num_warps)
34
+ for BK in [32, 64]
35
+ for num_warps in [1, 2, 4]
36
+ ],
37
+ key=["H", "Hg", "K", "BC"],
38
+ **autotune_cache_kwargs,
39
+ )
40
+ @triton.jit(do_not_specialize=["T"])
41
+ def chunk_gated_delta_rule_fwd_kkt_solve_kernel(
42
+ k: tl.tensor,
43
+ g: tl.tensor,
44
+ beta: tl.tensor,
45
+ A: tl.tensor,
46
+ cu_seqlens: tl.tensor,
47
+ chunk_indices: tl.tensor,
48
+ T: tl.int32,
49
+ H: tl.constexpr,
50
+ Hg: tl.constexpr,
51
+ K: tl.constexpr,
52
+ BT: tl.constexpr,
53
+ BC: tl.constexpr,
54
+ BK: tl.constexpr,
55
+ USE_G: tl.constexpr,
56
+ IS_VARLEN: tl.constexpr,
57
+ ) -> None:
58
+ """
59
+ Fused kernel: compute beta * K @ K^T (lower triangular) + solve_tril (I+A)^{-1} in one pass.
60
+
61
+ This kernel fuses chunk_scaled_dot_kkt_fwd and solve_tril into a single kernel,
62
+ avoiding the HBM round-trip for the intermediate A matrix.
63
+
64
+ Steps:
65
+ 1. Compute all 10 lower-triangular [BC, BC] blocks of beta * K @ K^T in registers
66
+ 2. Apply gate and beta scaling
67
+ 3. Forward substitution on diagonal blocks
68
+ 4. Block merge to get full (I+A)^{-1}
69
+ 5. Write result to A (output)
70
+ """
71
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
72
+ i_b, i_h = i_bh // H, i_bh % H
73
+
74
+ if IS_VARLEN:
75
+ i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
76
+ chunk_indices + i_t * 2 + 1
77
+ ).to(tl.int32)
78
+ bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
79
+ cu_seqlens + i_n + 1
80
+ ).to(tl.int32)
81
+ T = eos - bos
82
+ else:
83
+ bos, eos = i_b * T, i_b * T + T
84
+
85
+ if i_t * BT >= T:
86
+ return
87
+
88
+ i_tc0 = i_t * BT
89
+ i_tc1 = i_t * BT + BC
90
+ i_tc2 = i_t * BT + 2 * BC
91
+ i_tc3 = i_t * BT + 3 * BC
92
+
93
+ k += (bos * Hg + i_h // (H // Hg)) * K
94
+ A += (bos * H + i_h) * BT
95
+
96
+ o_i = tl.arange(0, BC)
97
+ m_tc0 = (i_tc0 + o_i) < T
98
+ m_tc1 = (i_tc1 + o_i) < T
99
+ m_tc2 = (i_tc2 + o_i) < T
100
+ m_tc3 = (i_tc3 + o_i) < T
101
+
102
+ # load beta for each sub-chunk
103
+ p_b0 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc0,), (BC,), (0,))
104
+ p_b1 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc1,), (BC,), (0,))
105
+ p_b2 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc2,), (BC,), (0,))
106
+ p_b3 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc3,), (BC,), (0,))
107
+ b_b0 = tl.load(p_b0, boundary_check=(0,)).to(tl.float32)
108
+ b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
109
+ b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
110
+ b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
111
+
112
+ # load gate if used
113
+ if USE_G:
114
+ p_g0 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc0,), (BC,), (0,))
115
+ p_g1 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc1,), (BC,), (0,))
116
+ p_g2 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc2,), (BC,), (0,))
117
+ p_g3 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc3,), (BC,), (0,))
118
+
119
+ b_g0 = tl.load(p_g0, boundary_check=(0,)).to(tl.float32)
120
+ b_g1 = tl.load(p_g1, boundary_check=(0,)).to(tl.float32)
121
+ b_g2 = tl.load(p_g2, boundary_check=(0,)).to(tl.float32)
122
+ b_g3 = tl.load(p_g3, boundary_check=(0,)).to(tl.float32)
123
+
124
+ ############################################################################
125
+ # Step 1: compute all 10 lower-triangular [BC, BC] blocks of K @ K^T
126
+ ############################################################################
127
+
128
+ # 4 diagonal blocks
129
+ b_A00 = tl.zeros([BC, BC], dtype=tl.float32)
130
+ b_A11 = tl.zeros([BC, BC], dtype=tl.float32)
131
+ b_A22 = tl.zeros([BC, BC], dtype=tl.float32)
132
+ b_A33 = tl.zeros([BC, BC], dtype=tl.float32)
133
+
134
+ # 6 off-diagonal blocks
135
+ b_A10 = tl.zeros([BC, BC], dtype=tl.float32)
136
+ b_A20 = tl.zeros([BC, BC], dtype=tl.float32)
137
+ b_A21 = tl.zeros([BC, BC], dtype=tl.float32)
138
+ b_A30 = tl.zeros([BC, BC], dtype=tl.float32)
139
+ b_A31 = tl.zeros([BC, BC], dtype=tl.float32)
140
+ b_A32 = tl.zeros([BC, BC], dtype=tl.float32)
141
+
142
+ for i_k in range(tl.cdiv(K, BK)):
143
+ p_k0 = tl.make_block_ptr(
144
+ k, (T, K), (Hg * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
145
+ )
146
+ b_k0 = tl.load(p_k0, boundary_check=(0, 1))
147
+ # diagonal block 0
148
+ b_A00 += tl.dot(b_k0, tl.trans(b_k0))
149
+
150
+ if i_tc1 < T:
151
+ p_k1 = tl.make_block_ptr(
152
+ k, (T, K), (Hg * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
153
+ )
154
+ b_k1 = tl.load(p_k1, boundary_check=(0, 1))
155
+ # diagonal block 1
156
+ b_A11 += tl.dot(b_k1, tl.trans(b_k1))
157
+ # off-diagonal (1,0)
158
+ b_A10 += tl.dot(b_k1, tl.trans(b_k0))
159
+
160
+ if i_tc2 < T:
161
+ p_k2 = tl.make_block_ptr(
162
+ k, (T, K), (Hg * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
163
+ )
164
+ b_k2 = tl.load(p_k2, boundary_check=(0, 1))
165
+ # diagonal block 2
166
+ b_A22 += tl.dot(b_k2, tl.trans(b_k2))
167
+ # off-diagonal (2,0), (2,1)
168
+ b_A20 += tl.dot(b_k2, tl.trans(b_k0))
169
+ b_A21 += tl.dot(b_k2, tl.trans(b_k1))
170
+
171
+ if i_tc3 < T:
172
+ p_k3 = tl.make_block_ptr(
173
+ k, (T, K), (Hg * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
174
+ )
175
+ b_k3 = tl.load(p_k3, boundary_check=(0, 1))
176
+ # diagonal block 3
177
+ b_A33 += tl.dot(b_k3, tl.trans(b_k3))
178
+ # off-diagonal (3,0), (3,1), (3,2)
179
+ b_A30 += tl.dot(b_k3, tl.trans(b_k0))
180
+ b_A31 += tl.dot(b_k3, tl.trans(b_k1))
181
+ b_A32 += tl.dot(b_k3, tl.trans(b_k2))
182
+
183
+ ############################################################################
184
+ # Step 2: apply gate and beta scaling
185
+ ############################################################################
186
+
187
+ if USE_G:
188
+ # diagonal blocks: g_diff = g_i - g_j within sub-chunk
189
+ b_A00 *= safe_exp(b_g0[:, None] - b_g0[None, :])
190
+ b_A11 *= safe_exp(b_g1[:, None] - b_g1[None, :])
191
+ b_A22 *= safe_exp(b_g2[:, None] - b_g2[None, :])
192
+ b_A33 *= safe_exp(b_g3[:, None] - b_g3[None, :])
193
+
194
+ # off-diagonal blocks: g_diff = g_row - g_col (cross sub-chunk)
195
+ b_A10 *= safe_exp(b_g1[:, None] - b_g0[None, :])
196
+ b_A20 *= safe_exp(b_g2[:, None] - b_g0[None, :])
197
+ b_A21 *= safe_exp(b_g2[:, None] - b_g1[None, :])
198
+ b_A30 *= safe_exp(b_g3[:, None] - b_g0[None, :])
199
+ b_A31 *= safe_exp(b_g3[:, None] - b_g1[None, :])
200
+ b_A32 *= safe_exp(b_g3[:, None] - b_g2[None, :])
201
+
202
+ # apply beta to row dimension and mask
203
+ m_d = o_i[:, None] > o_i[None, :]
204
+ m_I = o_i[:, None] == o_i[None, :]
205
+
206
+ # diagonal blocks: strictly lower triangular within sub-chunk, scaled by beta
207
+ b_A00 = (
208
+ tl.where(m_d & (m_tc0[:, None] & m_tc0[None, :]), b_A00, 0.0) * b_b0[:, None]
209
+ )
210
+ b_A11 = (
211
+ tl.where(m_d & (m_tc1[:, None] & m_tc1[None, :]), b_A11, 0.0) * b_b1[:, None]
212
+ )
213
+ b_A22 = (
214
+ tl.where(m_d & (m_tc2[:, None] & m_tc2[None, :]), b_A22, 0.0) * b_b2[:, None]
215
+ )
216
+ b_A33 = (
217
+ tl.where(m_d & (m_tc3[:, None] & m_tc3[None, :]), b_A33, 0.0) * b_b3[:, None]
218
+ )
219
+
220
+ # off-diagonal blocks: full block, scaled by beta
221
+ b_A10 = b_A10 * b_b1[:, None]
222
+ b_A20 = b_A20 * b_b2[:, None]
223
+ b_A21 = b_A21 * b_b2[:, None]
224
+ b_A30 = b_A30 * b_b3[:, None]
225
+ b_A31 = b_A31 * b_b3[:, None]
226
+ b_A32 = b_A32 * b_b3[:, None]
227
+
228
+ ############################################################################
229
+ # Step 3: forward substitution on diagonal blocks -> (I + A_diag)^{-1}
230
+ #
231
+ # Same algorithm as solve_tril, but rows are extracted from in-register
232
+ # [BC, BC] tensor via tl.sum(tl.where(mask, tensor, 0), 0) instead of
233
+ # tl.load from HBM.
234
+ ############################################################################
235
+
236
+ b_Ai00 = -b_A00
237
+ b_Ai11 = -b_A11
238
+ b_Ai22 = -b_A22
239
+ b_Ai33 = -b_A33
240
+
241
+ for i in range(2, min(BC, T - i_tc0)):
242
+ b_a00 = tl.sum(tl.where((o_i == i)[:, None], -b_A00, 0.0), 0)
243
+ b_a00 = tl.where(o_i < i, b_a00, 0.0)
244
+ b_a00 = b_a00 + tl.sum(b_a00[:, None] * b_Ai00, 0)
245
+ b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
246
+ for i in range(2, min(BC, T - i_tc1)):
247
+ b_a11 = tl.sum(tl.where((o_i == i)[:, None], -b_A11, 0.0), 0)
248
+ b_a11 = tl.where(o_i < i, b_a11, 0.0)
249
+ b_a11 = b_a11 + tl.sum(b_a11[:, None] * b_Ai11, 0)
250
+ b_Ai11 = tl.where((o_i == i)[:, None], b_a11, b_Ai11)
251
+ for i in range(2, min(BC, T - i_tc2)):
252
+ b_a22 = tl.sum(tl.where((o_i == i)[:, None], -b_A22, 0.0), 0)
253
+ b_a22 = tl.where(o_i < i, b_a22, 0.0)
254
+ b_a22 = b_a22 + tl.sum(b_a22[:, None] * b_Ai22, 0)
255
+ b_Ai22 = tl.where((o_i == i)[:, None], b_a22, b_Ai22)
256
+ for i in range(2, min(BC, T - i_tc3)):
257
+ b_a33 = tl.sum(tl.where((o_i == i)[:, None], -b_A33, 0.0), 0)
258
+ b_a33 = tl.where(o_i < i, b_a33, 0.0)
259
+ b_a33 = b_a33 + tl.sum(b_a33[:, None] * b_Ai33, 0)
260
+ b_Ai33 = tl.where((o_i == i)[:, None], b_a33, b_Ai33)
261
+
262
+ b_Ai00 += m_I
263
+ b_Ai11 += m_I
264
+ b_Ai22 += m_I
265
+ b_Ai33 += m_I
266
+
267
+ ############################################################################
268
+ # Step 4: block merge -> full (I + A)^{-1}
269
+ ############################################################################
270
+
271
+ b_Ai10 = -tl.dot(
272
+ tl.dot(b_Ai11, b_A10, input_precision=_MERGE_DOT_PRECISION),
273
+ b_Ai00,
274
+ input_precision=_MERGE_DOT_PRECISION,
275
+ )
276
+ b_Ai21 = -tl.dot(
277
+ tl.dot(b_Ai22, b_A21, input_precision=_MERGE_DOT_PRECISION),
278
+ b_Ai11,
279
+ input_precision=_MERGE_DOT_PRECISION,
280
+ )
281
+ b_Ai32 = -tl.dot(
282
+ tl.dot(b_Ai33, b_A32, input_precision=_MERGE_DOT_PRECISION),
283
+ b_Ai22,
284
+ input_precision=_MERGE_DOT_PRECISION,
285
+ )
286
+
287
+ b_Ai20 = -tl.dot(
288
+ b_Ai22,
289
+ tl.dot(b_A20, b_Ai00, input_precision=_MERGE_DOT_PRECISION)
290
+ + tl.dot(b_A21, b_Ai10, input_precision=_MERGE_DOT_PRECISION),
291
+ input_precision=_MERGE_DOT_PRECISION,
292
+ )
293
+ b_Ai31 = -tl.dot(
294
+ b_Ai33,
295
+ tl.dot(b_A31, b_Ai11, input_precision=_MERGE_DOT_PRECISION)
296
+ + tl.dot(b_A32, b_Ai21, input_precision=_MERGE_DOT_PRECISION),
297
+ input_precision=_MERGE_DOT_PRECISION,
298
+ )
299
+ b_Ai30 = -tl.dot(
300
+ b_Ai33,
301
+ tl.dot(b_A30, b_Ai00, input_precision=_MERGE_DOT_PRECISION)
302
+ + tl.dot(b_A31, b_Ai10, input_precision=_MERGE_DOT_PRECISION)
303
+ + tl.dot(b_A32, b_Ai20, input_precision=_MERGE_DOT_PRECISION),
304
+ input_precision=_MERGE_DOT_PRECISION,
305
+ )
306
+
307
+ ############################################################################
308
+ # Step 5: store full (I + A)^{-1} to output A
309
+ ############################################################################
310
+
311
+ p_A00 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
312
+ p_A10 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
313
+ p_A11 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
314
+ p_A20 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
315
+ p_A21 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
316
+ p_A22 = tl.make_block_ptr(
317
+ A, (T, BT), (H * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
318
+ )
319
+ p_A30 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
320
+ p_A31 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
321
+ p_A32 = tl.make_block_ptr(
322
+ A, (T, BT), (H * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
323
+ )
324
+ p_A33 = tl.make_block_ptr(
325
+ A, (T, BT), (H * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
326
+ )
327
+
328
+ tl.store(p_A00, b_Ai00.to(A.dtype.element_ty), boundary_check=(0, 1))
329
+ tl.store(p_A10, b_Ai10.to(A.dtype.element_ty), boundary_check=(0, 1))
330
+ tl.store(p_A11, b_Ai11.to(A.dtype.element_ty), boundary_check=(0, 1))
331
+ tl.store(p_A20, b_Ai20.to(A.dtype.element_ty), boundary_check=(0, 1))
332
+ tl.store(p_A21, b_Ai21.to(A.dtype.element_ty), boundary_check=(0, 1))
333
+ tl.store(p_A22, b_Ai22.to(A.dtype.element_ty), boundary_check=(0, 1))
334
+ tl.store(p_A30, b_Ai30.to(A.dtype.element_ty), boundary_check=(0, 1))
335
+ tl.store(p_A31, b_Ai31.to(A.dtype.element_ty), boundary_check=(0, 1))
336
+ tl.store(p_A32, b_Ai32.to(A.dtype.element_ty), boundary_check=(0, 1))
337
+ tl.store(p_A33, b_Ai33.to(A.dtype.element_ty), boundary_check=(0, 1))
338
+
339
+
340
+ def chunk_gated_delta_rule_fwd_intra(
341
+ k: torch.Tensor,
342
+ v: torch.Tensor,
343
+ g: torch.Tensor | None = None,
344
+ beta: torch.Tensor | None = None,
345
+ cu_seqlens: torch.LongTensor | None = None,
346
+ chunk_size: int = 64,
347
+ chunk_indices: torch.LongTensor | None = None,
348
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
349
+ r"""
350
+ GDN intra-chunk forward: fused kkt + solve_tril + recompute_w_u.
351
+
352
+ Equivalent to:
353
+ A = chunk_scaled_dot_kkt_fwd(k, g, beta, ...) # kernel 1
354
+ A = solve_tril(A, ...) # kernel 2
355
+ w, u = recompute_w_u_fwd(k, v, beta, A, g, ...) # kernel 3
356
+
357
+ Fuses kernels 1+2 into a single kernel, reducing from 3 to 2 kernel launches
358
+ and eliminating the HBM round-trip for the intermediate A matrix.
359
+
360
+ Args:
361
+ k (torch.Tensor):
362
+ The key tensor of shape `[B, T, H, K]`.
363
+ v (torch.Tensor):
364
+ The value tensor of shape `[B, T, H, V]`.
365
+ g (torch.Tensor):
366
+ The cumulative sum of the gate tensor of shape `[B, T, H]`. Default: `None`.
367
+ beta (torch.Tensor):
368
+ The beta tensor of shape `[B, T, H]`.
369
+ cu_seqlens (torch.LongTensor):
370
+ The cumulative sequence lengths. Default: `None`.
371
+ chunk_size (int):
372
+ The chunk size. Default: 64.
373
+ chunk_indices (torch.LongTensor):
374
+ Precomputed chunk indices. Default: `None`.
375
+
376
+ Returns:
377
+ w (torch.Tensor): shape `[B, T, H, K]`
378
+ u (torch.Tensor): shape `[B, T, H, V]`
379
+ A (torch.Tensor): shape `[B, T, H, BT]`, the solved (I+A)^{-1} matrix
380
+ """
381
+ B, T, Hg, K = k.shape
382
+ H = beta.shape[-1]
383
+ BT = chunk_size
384
+ BC = 16
385
+
386
+ if chunk_indices is None and cu_seqlens is not None:
387
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
388
+ NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
389
+
390
+ # Step 1: fused kkt + solve_tril
391
+ A = torch.zeros(B, T, H, BT, device=k.device, dtype=k.dtype)
392
+ chunk_gated_delta_rule_fwd_kkt_solve_kernel[(NT, B * H)](
393
+ k=k,
394
+ g=g,
395
+ beta=beta,
396
+ A=A,
397
+ cu_seqlens=cu_seqlens,
398
+ chunk_indices=chunk_indices,
399
+ T=T,
400
+ H=H,
401
+ Hg=Hg,
402
+ K=K,
403
+ BT=BT,
404
+ BC=BC,
405
+ )
406
+
407
+ # Step 2: recompute_w_u
408
+ w, u = recompute_w_u_fwd(
409
+ k=k,
410
+ v=v,
411
+ beta=beta,
412
+ A=A,
413
+ g_cumsum=g,
414
+ cu_seqlens=cu_seqlens,
415
+ chunk_indices=chunk_indices,
416
+ )
417
+ return w, u, A
clairic_runtime/gdn/chunk_o.py ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_o.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+ import triton
10
+ import triton.language as tl
11
+
12
+ from .index import prepare_chunk_indices
13
+ from .op import exp, safe_exp
14
+ from .utils import check_shared_mem, is_nvidia_hopper
15
+
16
+ BKV_LIST = [64, 128] if check_shared_mem() else [32, 64]
17
+ NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8]
18
+
19
+
20
+ # @triton.autotune(
21
+ # configs=[
22
+ # triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)
23
+ # for BK in BKV_LIST
24
+ # for BV in BKV_LIST
25
+ # for num_warps in NUM_WARPS
26
+ # for num_stages in [2, 3, 4]
27
+ # ],
28
+ # key=["H", "K", "V", "BT"],
29
+ # )
30
+ @triton.jit(do_not_specialize=["T"])
31
+ def chunk_fwd_kernel_o(
32
+ q: tl.tensor,
33
+ k: tl.tensor,
34
+ v: tl.tensor,
35
+ h: tl.tensor,
36
+ g: tl.tensor,
37
+ o: tl.tensor,
38
+ cu_seqlens: tl.tensor,
39
+ chunk_indices: tl.tensor,
40
+ scale: tl.float32,
41
+ T: tl.int32,
42
+ H: tl.constexpr,
43
+ Hg: tl.constexpr,
44
+ K: tl.constexpr,
45
+ V: tl.constexpr,
46
+ BT: tl.constexpr,
47
+ BK: tl.constexpr,
48
+ BV: tl.constexpr,
49
+ USE_G: tl.constexpr,
50
+ IS_VARLEN: tl.constexpr,
51
+ ) -> None:
52
+ i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
53
+ i_b, i_h = i_bh // H, i_bh % H
54
+
55
+ if IS_VARLEN:
56
+ i_tg = i_t
57
+ i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
58
+ chunk_indices + i_t * 2 + 1
59
+ ).to(tl.int32)
60
+ bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
61
+ cu_seqlens + i_n + 1
62
+ ).to(tl.int32)
63
+ T = eos - bos
64
+ NT = tl.cdiv(T, BT)
65
+ else:
66
+ NT = tl.cdiv(T, BT)
67
+ i_tg = i_b * NT + i_t
68
+ bos, eos = i_b * T, i_b * T + T
69
+
70
+ # offset calculation
71
+ q += (bos * Hg + i_h // (H // Hg)) * K
72
+ k += (bos * Hg + i_h // (H // Hg)) * K
73
+ v += (bos * H + i_h) * V
74
+ o += (bos * H + i_h) * V
75
+ h += (i_tg * H + i_h).to(tl.int64) * V * K
76
+
77
+ b_o = tl.zeros([BT, BV], dtype=tl.float32)
78
+ b_A = tl.zeros([BT, BT], dtype=tl.float32)
79
+
80
+ for i_k in range(tl.cdiv(K, BK)):
81
+ p_q = tl.make_block_ptr(
82
+ q, (T, K), (Hg * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)
83
+ )
84
+ p_k = tl.make_block_ptr(
85
+ k, (K, T), (1, Hg * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1)
86
+ )
87
+ p_h = tl.make_block_ptr(
88
+ h, (V, K), (K, 1), (i_v * BV, i_k * BK), (BV, BK), (1, 0)
89
+ )
90
+ # [BT, BK]
91
+ b_q = tl.load(p_q, boundary_check=(0, 1))
92
+ # [BK, BT]
93
+ b_k = tl.load(p_k, boundary_check=(0, 1))
94
+ # [BV, BK]
95
+ b_h = tl.load(p_h, boundary_check=(0, 1))
96
+
97
+ # [BT, BK] @ [BK, BV] -> [BT, BV]
98
+ b_o += tl.dot(b_q, tl.trans(b_h))
99
+ # [BT, BK] @ [BK, BT] -> [BT, BT]
100
+ b_A += tl.dot(b_q, b_k)
101
+
102
+ if USE_G:
103
+ g += bos * H + i_h
104
+ p_g = tl.make_block_ptr(g, (T,), (H,), (i_t * BT,), (BT,), (0,))
105
+ b_g = tl.load(p_g, boundary_check=(0,))
106
+ b_o = b_o * exp(b_g)[:, None]
107
+ b_A = b_A * safe_exp(b_g[:, None] - b_g[None, :])
108
+
109
+ o_i = tl.arange(0, BT)
110
+ m_A = o_i[:, None] >= o_i[None, :]
111
+ b_A = tl.where(m_A, b_A, 0)
112
+
113
+ p_v = tl.make_block_ptr(
114
+ v, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
115
+ )
116
+ p_o = tl.make_block_ptr(
117
+ o, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
118
+ )
119
+ b_v = tl.load(p_v, boundary_check=(0, 1))
120
+
121
+ # to fix mma -> mma layout conversion
122
+ # already solved by triton v3.2 or higher
123
+ b_o = b_o * scale + tl.dot(b_A.to(b_v.dtype), b_v) * scale
124
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
125
+
126
+
127
+ def chunk_fwd_o(
128
+ q: torch.Tensor,
129
+ k: torch.Tensor,
130
+ v: torch.Tensor,
131
+ h: torch.Tensor,
132
+ g: Optional[torch.Tensor] = None, # cumsum of log decay
133
+ scale: Optional[float] = None,
134
+ cu_seqlens: Optional[torch.LongTensor] = None,
135
+ chunk_size: int = 64,
136
+ ) -> torch.Tensor:
137
+ B, T, Hg, K, V = *q.shape, v.shape[-1]
138
+ H = v.shape[-2]
139
+ BT = min(chunk_size, max(16, triton.next_power_of_2(T)))
140
+ chunk_indices = (
141
+ prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
142
+ )
143
+ NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
144
+ if scale is None:
145
+ scale = k.shape[-1] ** -0.5
146
+
147
+ o = torch.zeros_like(v)
148
+
149
+ def grid(meta: dict[str, int]) -> tuple[int, int, int]:
150
+ return (triton.cdiv(V, meta["BV"]), NT, B * H)
151
+
152
+ chunk_fwd_kernel_o[grid](
153
+ q,
154
+ k,
155
+ v,
156
+ h,
157
+ g,
158
+ o,
159
+ cu_seqlens,
160
+ chunk_indices,
161
+ scale,
162
+ T=T,
163
+ H=H,
164
+ Hg=Hg,
165
+ K=K,
166
+ V=V,
167
+ BT=BT,
168
+ BK=128,
169
+ BV=64,
170
+ USE_G=g is not None,
171
+ IS_VARLEN=cu_seqlens is not None,
172
+ num_warps=4,
173
+ num_stages=2,
174
+ )
175
+ return o
clairic_runtime/gdn/cumsum.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/cumsum.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+ import triton
10
+ import triton.language as tl
11
+
12
+ from .index import prepare_chunk_indices
13
+
14
+
15
+ @triton.jit(do_not_specialize=["T"])
16
+ def chunk_local_cumsum_scalar_kernel(
17
+ s: tl.tensor,
18
+ o: tl.tensor,
19
+ scale: tl.tensor,
20
+ cu_seqlens: tl.tensor,
21
+ chunk_indices: tl.tensor,
22
+ T: tl.int32,
23
+ B: tl.constexpr,
24
+ H: tl.constexpr,
25
+ BT: tl.constexpr,
26
+ REVERSE: tl.constexpr,
27
+ HAS_SCALE: tl.constexpr,
28
+ IS_VARLEN: tl.constexpr,
29
+ HEAD_FIRST: tl.constexpr,
30
+ ) -> None:
31
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
32
+ i_b, i_h = i_bh // H, i_bh % H
33
+ if IS_VARLEN:
34
+ i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
35
+ chunk_indices + i_t * 2 + 1
36
+ ).to(tl.int32)
37
+ bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
38
+ cu_seqlens + i_n + 1
39
+ ).to(tl.int32)
40
+ T = eos - bos
41
+ else:
42
+ bos, eos = i_b * T, i_b * T + T
43
+
44
+ if HEAD_FIRST:
45
+ p_s = tl.make_block_ptr(
46
+ s + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,)
47
+ )
48
+ p_o = tl.make_block_ptr(
49
+ o + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,)
50
+ )
51
+ else:
52
+ p_s = tl.make_block_ptr(s + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
53
+ p_o = tl.make_block_ptr(o + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
54
+ # [BT]
55
+ b_s = tl.load(p_s, boundary_check=(0,)).to(tl.float32)
56
+ b_o = tl.cumsum(b_s, axis=0)
57
+ if REVERSE:
58
+ b_z = tl.sum(b_s, axis=0)
59
+ b_o = -b_o + b_z[None] + b_s
60
+ if HAS_SCALE:
61
+ b_o *= scale
62
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0,))
63
+
64
+
65
+ def chunk_local_cumsum_scalar(
66
+ g: torch.Tensor,
67
+ chunk_size: int,
68
+ reverse: bool = False,
69
+ scale: float = None,
70
+ cu_seqlens: Optional[torch.Tensor] = None,
71
+ head_first: bool = False,
72
+ output_dtype: Optional[torch.dtype] = torch.float,
73
+ chunk_indices: Optional[torch.LongTensor] = None,
74
+ ) -> torch.Tensor:
75
+ if head_first:
76
+ B, H, T = g.shape
77
+ else:
78
+ B, T, H = g.shape
79
+ assert chunk_size == 2 ** (
80
+ chunk_size.bit_length() - 1
81
+ ), "chunk_size must be a power of 2"
82
+ BT = chunk_size
83
+ if chunk_indices is None and cu_seqlens is not None:
84
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
85
+ NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
86
+ g_org, g = g, torch.empty_like(g, dtype=output_dtype or g.dtype)
87
+ grid = (NT, B * H)
88
+ chunk_local_cumsum_scalar_kernel[grid](
89
+ s=g_org,
90
+ o=g,
91
+ scale=scale,
92
+ cu_seqlens=cu_seqlens,
93
+ chunk_indices=chunk_indices,
94
+ T=T,
95
+ B=B,
96
+ H=H,
97
+ BT=BT,
98
+ HEAD_FIRST=head_first,
99
+ REVERSE=reverse,
100
+ HAS_SCALE=scale is not None,
101
+ IS_VARLEN=cu_seqlens is not None,
102
+ num_warps=8,
103
+ num_stages=3,
104
+ )
105
+ return g
106
+
clairic_runtime/gdn/gating.py ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
2
+ # Inference-only adaptation; see LICENSES and NOTICE.
3
+ from typing import Tuple
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+
10
+ # g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
11
+ # beta_output = b.sigmoid()
12
+ @triton.jit
13
+ def fused_gdn_gating_kernel(
14
+ g: tl.tensor,
15
+ beta_output: tl.tensor,
16
+ A_log: tl.tensor,
17
+ a: tl.tensor,
18
+ b: tl.tensor,
19
+ dt_bias: tl.tensor,
20
+ seq_len: tl.int32,
21
+ stride_a: tl.int32,
22
+ stride_b: tl.int32,
23
+ NUM_HEADS: tl.constexpr,
24
+ beta: tl.constexpr,
25
+ threshold: tl.constexpr,
26
+ BLK_HEADS: tl.constexpr,
27
+ ) -> None:
28
+ i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
29
+ head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
30
+ off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
31
+ mask = head_off < NUM_HEADS
32
+ blk_A_log = tl.load(A_log + head_off, mask=mask)
33
+ blk_a = tl.load(a + i_b * stride_a + head_off, mask=mask)
34
+ blk_b = tl.load(b + i_b * stride_b + head_off, mask=mask)
35
+ blk_bias = tl.load(dt_bias + head_off, mask=mask)
36
+ x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
37
+ softplus_x = tl.where(
38
+ beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
39
+ )
40
+ blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
41
+ tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
42
+ blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
43
+ tl.store(beta_output + off, blk_beta_output.to(b.dtype.element_ty), mask=mask)
44
+
45
+
46
+ def fused_gdn_gating(
47
+ A_log: torch.Tensor,
48
+ a: torch.Tensor,
49
+ b: torch.Tensor,
50
+ dt_bias: torch.Tensor,
51
+ beta: float = 1.0,
52
+ threshold: float = 20.0,
53
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
54
+ batch, num_heads = a.shape
55
+ seq_len = 1
56
+ stride_a = a.stride(0)
57
+ stride_b = b.stride(0)
58
+ grid = (batch, seq_len, triton.cdiv(num_heads, 8))
59
+ g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
60
+ beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
61
+ fused_gdn_gating_kernel[grid](
62
+ g,
63
+ beta_output,
64
+ A_log,
65
+ a,
66
+ b,
67
+ dt_bias,
68
+ seq_len,
69
+ stride_a,
70
+ stride_b,
71
+ num_heads,
72
+ beta,
73
+ threshold,
74
+ 8,
75
+ num_warps=1,
76
+ )
77
+ return g, beta_output
clairic_runtime/gdn/index.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/index.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ import torch
7
+ import triton
8
+
9
+ from .utils import tensor_cache
10
+
11
+
12
+ @tensor_cache
13
+ def prepare_lens(cu_seqlens: torch.LongTensor) -> torch.LongTensor:
14
+ return cu_seqlens[1:] - cu_seqlens[:-1]
15
+
16
+
17
+ @tensor_cache
18
+ def prepare_chunk_indices(
19
+ cu_seqlens: torch.LongTensor, chunk_size: int
20
+ ) -> torch.LongTensor:
21
+ indices = torch.cat(
22
+ [
23
+ torch.arange(n)
24
+ for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()
25
+ ]
26
+ )
27
+ return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens)
28
+
29
+
30
+ @tensor_cache
31
+ def prepare_chunk_offsets(
32
+ cu_seqlens: torch.LongTensor, chunk_size: int
33
+ ) -> torch.LongTensor:
34
+ return torch.cat(
35
+ [cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]
36
+ ).cumsum(-1)
clairic_runtime/gdn/l2norm.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/modules/l2norm.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+ import triton
10
+ import triton.language as tl
11
+
12
+ @triton.jit
13
+ def l2norm_fwd_kernel1(
14
+ x: tl.tensor,
15
+ y: tl.tensor,
16
+ D: tl.int32,
17
+ BD: tl.constexpr,
18
+ eps: tl.float32,
19
+ ) -> None:
20
+ i_t = tl.program_id(0)
21
+ x += i_t * D
22
+ y += i_t * D
23
+ # Compute mean and variance
24
+ cols = tl.arange(0, BD)
25
+ mask = cols < D
26
+ b_x = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32)
27
+ b_var = tl.sum(b_x * b_x, axis=0)
28
+ b_rstd = 1 / tl.sqrt(b_var + eps)
29
+ # tl.store(Rstd + i_t, rstd)
30
+ # Normalize and apply linear transformation
31
+ b_y = b_x * b_rstd
32
+ tl.store(y + cols, b_y, mask=mask)
33
+
34
+
35
+ @triton.jit(do_not_specialize=["T"])
36
+ def l2norm_fwd_kernel(
37
+ x: tl.tensor,
38
+ y: tl.tensor,
39
+ eps: tl.float32,
40
+ T: tl.int32,
41
+ D: tl.constexpr,
42
+ BT: tl.constexpr,
43
+ BD: tl.constexpr,
44
+ ) -> None:
45
+ i_t = tl.program_id(0)
46
+ p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
47
+ b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
48
+ b_var = tl.sum(b_x * b_x, axis=1)
49
+ b_y = b_x / tl.sqrt(b_var + eps)[:, None]
50
+ p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
51
+ tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
52
+
53
+
54
+ def l2norm_fwd(
55
+ x: torch.Tensor, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None
56
+ ) -> torch.Tensor:
57
+ x_shape_og = x.shape
58
+ x = x.view(-1, x.shape[-1])
59
+ # allocate output
60
+ if output_dtype is None:
61
+ y = torch.empty_like(x)
62
+ else:
63
+ y = torch.empty_like(x, dtype=output_dtype)
64
+ assert y.stride(-1) == 1
65
+ T, D = x.shape[0], x.shape[-1]
66
+ # rstd = torch.empty((T,), dtype=torch.float32, device=x.device)
67
+ # Less than 64KB per feature: enqueue fused kernel
68
+ MAX_FUSED_SIZE = 65536 // x.element_size()
69
+ BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
70
+ if D > BD:
71
+ raise RuntimeError("This layer doesn't support feature dim >= 64KB.")
72
+
73
+ if D <= 512:
74
+
75
+ def grid(meta: dict[str, int]) -> tuple[int]:
76
+ return (triton.cdiv(T, meta["BT"]),)
77
+
78
+ l2norm_fwd_kernel[grid](
79
+ x,
80
+ y,
81
+ eps,
82
+ T=T,
83
+ D=D,
84
+ BD=BD,
85
+ BT=16,
86
+ num_warps=8,
87
+ num_stages=3,
88
+ )
89
+ else:
90
+ l2norm_fwd_kernel1[(T,)](
91
+ x,
92
+ y,
93
+ eps=eps,
94
+ D=D,
95
+ BD=BD,
96
+ num_warps=8,
97
+ num_stages=3,
98
+ )
99
+
100
+ return y.view(x_shape_og)
101
+
clairic_runtime/gdn/op.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/op.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ import os
7
+
8
+ import triton
9
+ import triton.language as tl
10
+ import triton.language.extra.libdevice as tldevice
11
+
12
+ if os.environ.get("FLA_USE_FAST_OPS", "0") == "1":
13
+ exp = tldevice.fast_expf
14
+ exp2 = tldevice.exp2
15
+ log = tldevice.fast_logf
16
+ log2 = tldevice.fast_log2f
17
+ else:
18
+ exp = tl.exp
19
+ exp2 = tl.math.exp2
20
+ log = tl.log
21
+ log2 = tl.log2
22
+
23
+
24
+
25
+ @triton.jit
26
+ def safe_exp(x: tl.tensor) -> tl.tensor:
27
+ return exp(tl.where(x <= 0, x, float("-inf")))
28
+
clairic_runtime/gdn/prefill.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any
2
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
3
+ # Inference-only adaptation; see LICENSES and NOTICE.
4
+ from typing import Optional
5
+ import torch
6
+ from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
7
+ from .chunk_fwd import chunk_gated_delta_rule_fwd_intra
8
+ from .chunk_o import chunk_fwd_o
9
+ from .cumsum import chunk_local_cumsum_scalar as chunk_local_cumsum
10
+ CHUNK_SIZE = 64
11
+ SUPPRESS_LEVEL = 0
12
+
13
+ def chunk_gated_delta_rule_fwd(
14
+ q: torch.Tensor,
15
+ k: torch.Tensor,
16
+ v: torch.Tensor,
17
+ g: torch.Tensor,
18
+ beta: torch.Tensor,
19
+ scale: float,
20
+ initial_state: torch.Tensor,
21
+ initial_state_indices: torch.Tensor,
22
+ cu_seqlens: Optional[torch.LongTensor] = None,
23
+ chunk_indices: torch.LongTensor | None = None,
24
+ ) -> Any:
25
+ g = chunk_local_cumsum(
26
+ g, chunk_size=CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
27
+ )
28
+
29
+ # fused kkt + solve_tril + recompute_w_u
30
+ w, u, A = chunk_gated_delta_rule_fwd_intra(
31
+ k=k,
32
+ v=v,
33
+ g=g,
34
+ beta=beta,
35
+ cu_seqlens=cu_seqlens,
36
+ chunk_indices=chunk_indices,
37
+ )
38
+
39
+ h, v_new = chunk_gated_delta_rule_fwd_h(
40
+ k=k,
41
+ w=w,
42
+ u=u,
43
+ g=g,
44
+ initial_state=initial_state,
45
+ initial_state_indices=initial_state_indices,
46
+ cu_seqlens=cu_seqlens,
47
+ chunk_indices=chunk_indices,
48
+ )
49
+ o = chunk_fwd_o(
50
+ q=q,
51
+ k=k,
52
+ v=v_new,
53
+ h=h,
54
+ g=g,
55
+ scale=scale,
56
+ cu_seqlens=cu_seqlens,
57
+ )
58
+ if SUPPRESS_LEVEL < 3:
59
+ return g, o, A, None, h, None
60
+ elif SUPPRESS_LEVEL >= 3:
61
+ return g, o, A, w, h, v_new
62
+
clairic_runtime/gdn/recurrent.py ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
2
+ # Inference-only adaptation; see LICENSES and NOTICE.
3
+ import torch
4
+ import triton
5
+ import triton.language as tl
6
+ from .op import exp
7
+
8
+ @triton.jit
9
+ def fused_recurrent_gated_delta_rule_packed_decode_kernel(
10
+ mixed_qkv: tl.tensor,
11
+ a: tl.tensor,
12
+ b: tl.tensor,
13
+ A_log: tl.tensor,
14
+ dt_bias: tl.tensor,
15
+ o: tl.tensor,
16
+ h0: tl.tensor,
17
+ ht: tl.tensor,
18
+ ssm_state_indices: tl.tensor,
19
+ scale: tl.float32,
20
+ stride_mixed_qkv_tok: tl.constexpr,
21
+ stride_a_tok: tl.constexpr,
22
+ stride_b_tok: tl.constexpr,
23
+ stride_init_state_token: tl.constexpr,
24
+ stride_final_state_token: tl.constexpr,
25
+ stride_indices_seq: tl.constexpr,
26
+ H: tl.constexpr,
27
+ HV: tl.constexpr,
28
+ K: tl.constexpr,
29
+ V: tl.constexpr,
30
+ BK: tl.constexpr,
31
+ BV: tl.constexpr,
32
+ SOFTPLUS_THRESHOLD: tl.constexpr,
33
+ USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
34
+ ) -> None:
35
+ i_v, i_nh = tl.program_id(0), tl.program_id(1)
36
+ i_n, i_hv = i_nh // HV, i_nh % HV
37
+ i_h = i_hv // (HV // H)
38
+
39
+ o_k = tl.arange(0, BK)
40
+ o_v = i_v * BV + tl.arange(0, BV)
41
+ mask_k = o_k < K
42
+ mask_v = o_v < V
43
+ mask_h = mask_v[:, None] & mask_k[None, :]
44
+
45
+ state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)
46
+ p_o = o + (i_n * HV + i_hv) * V + o_v
47
+
48
+ if state_idx < 0:
49
+ zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
50
+ tl.store(p_o, zero, mask=mask_v)
51
+ return
52
+
53
+ p_h0 = h0 + state_idx * stride_init_state_token
54
+ p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
55
+ b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
56
+
57
+ p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok
58
+ q_off = i_h * K + o_k
59
+ k_off = (H * K) + i_h * K + o_k
60
+ v_off = (2 * H * K) + i_hv * V + o_v
61
+ b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)
62
+ b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)
63
+ b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)
64
+
65
+ if USE_QK_L2NORM_IN_KERNEL:
66
+ b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
67
+ b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
68
+ b_q = b_q * scale
69
+
70
+ a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)
71
+ b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)
72
+ A_log_val = tl.load(A_log + i_hv).to(tl.float32)
73
+ dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)
74
+ x = a_val + dt_bias_val
75
+ softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
76
+ g_val = -tl.exp(A_log_val) * softplus_x
77
+ beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32)
78
+
79
+ b_h *= exp(g_val)
80
+ b_v -= tl.sum(b_h * b_k[None, :], 1)
81
+ b_v *= beta_val
82
+ b_h += b_v[:, None] * b_k[None, :]
83
+ b_o = tl.sum(b_h * b_q[None, :], 1)
84
+ tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
85
+
86
+ p_ht = ht + state_idx * stride_final_state_token
87
+ p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
88
+ tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
89
+
90
+
91
+ def fused_recurrent_gated_delta_rule_packed_decode(
92
+ mixed_qkv: torch.Tensor,
93
+ a: torch.Tensor,
94
+ b: torch.Tensor,
95
+ A_log: torch.Tensor,
96
+ dt_bias: torch.Tensor,
97
+ scale: float,
98
+ initial_state: torch.Tensor,
99
+ out: torch.Tensor,
100
+ ssm_state_indices: torch.Tensor,
101
+ use_qk_l2norm_in_kernel: bool = False,
102
+ ) -> tuple[torch.Tensor, torch.Tensor]:
103
+ if mixed_qkv.ndim != 2:
104
+ raise ValueError(
105
+ f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."
106
+ )
107
+ if mixed_qkv.stride(-1) != 1:
108
+ raise ValueError("`mixed_qkv` must be contiguous in the last dim.")
109
+ if a.ndim != 2 or b.ndim != 2:
110
+ raise ValueError(
111
+ f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."
112
+ )
113
+ if a.stride(-1) != 1 or b.stride(-1) != 1:
114
+ raise ValueError("`a`/`b` must be contiguous in the last dim.")
115
+ if A_log.ndim != 1 or dt_bias.ndim != 1:
116
+ raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")
117
+ if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:
118
+ raise ValueError("`A_log`/`dt_bias` must be contiguous.")
119
+ if ssm_state_indices.ndim != 1:
120
+ raise ValueError(
121
+ f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."
122
+ )
123
+ if not out.is_contiguous():
124
+ raise ValueError("`out` must be contiguous.")
125
+
126
+ dev = mixed_qkv.device
127
+ if any(
128
+ t.device != dev
129
+ for t in (a, b, A_log, dt_bias, initial_state, out, ssm_state_indices)
130
+ ):
131
+ raise ValueError("All inputs must be on the same device.")
132
+
133
+ B = mixed_qkv.shape[0]
134
+ if a.shape[0] != B or b.shape[0] != B:
135
+ raise ValueError(
136
+ "Mismatched batch sizes: "
137
+ f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."
138
+ )
139
+ if ssm_state_indices.shape[0] != B:
140
+ raise ValueError(
141
+ f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."
142
+ )
143
+
144
+ if initial_state.ndim != 4:
145
+ raise ValueError(
146
+ f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."
147
+ )
148
+ if initial_state.stride(-1) != 1:
149
+ raise ValueError("`initial_state` must be contiguous in the last dim.")
150
+ HV, V, K = initial_state.shape[-3:]
151
+ if a.shape[1] != HV or b.shape[1] != HV:
152
+ raise ValueError(
153
+ f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."
154
+ )
155
+ if A_log.numel() != HV or dt_bias.numel() != HV:
156
+ raise ValueError(
157
+ f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."
158
+ )
159
+ if out.shape != (B, 1, HV, V):
160
+ raise ValueError(
161
+ f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."
162
+ )
163
+
164
+ qkv_dim = mixed_qkv.shape[1]
165
+ qk_dim = qkv_dim - HV * V
166
+ if qk_dim <= 0 or qk_dim % 2 != 0:
167
+ raise ValueError(
168
+ f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."
169
+ )
170
+ q_dim = qk_dim // 2
171
+ if q_dim % K != 0:
172
+ raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")
173
+ H = q_dim // K
174
+ if H <= 0 or HV % H != 0:
175
+ raise ValueError(
176
+ f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."
177
+ )
178
+
179
+ BK = triton.next_power_of_2(K)
180
+ if triton.cdiv(K, BK) != 1:
181
+ raise ValueError(
182
+ f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
183
+ )
184
+ BV = min(triton.next_power_of_2(V), 32)
185
+ num_stages = 3
186
+ num_warps = 1
187
+
188
+ stride_mixed_qkv_tok = mixed_qkv.stride(0)
189
+ stride_a_tok = a.stride(0)
190
+ stride_b_tok = b.stride(0)
191
+ stride_init_state_token = initial_state.stride(0)
192
+ stride_final_state_token = initial_state.stride(0)
193
+ stride_indices_seq = ssm_state_indices.stride(0)
194
+
195
+ NV = triton.cdiv(V, BV)
196
+ grid = (NV, B * HV)
197
+ fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
198
+ mixed_qkv=mixed_qkv,
199
+ a=a,
200
+ b=b,
201
+ A_log=A_log,
202
+ dt_bias=dt_bias,
203
+ o=out,
204
+ h0=initial_state,
205
+ ht=initial_state,
206
+ ssm_state_indices=ssm_state_indices,
207
+ scale=scale,
208
+ stride_mixed_qkv_tok=stride_mixed_qkv_tok,
209
+ stride_a_tok=stride_a_tok,
210
+ stride_b_tok=stride_b_tok,
211
+ stride_init_state_token=stride_init_state_token,
212
+ stride_final_state_token=stride_final_state_token,
213
+ stride_indices_seq=stride_indices_seq,
214
+ H=H,
215
+ HV=HV,
216
+ K=K,
217
+ V=V,
218
+ BK=BK,
219
+ BV=BV,
220
+ SOFTPLUS_THRESHOLD=20.0,
221
+ USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
222
+ num_warps=num_warps,
223
+ num_stages=num_stages,
224
+ )
225
+ return out, initial_state
226
+
clairic_runtime/gdn/utils.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
2
+ # Standalone helper plumbing; see LICENSES and NOTICE for retained notices.
3
+
4
+ """Device, cache, and launch helpers for the standalone inference kernels."""
5
+
6
+ import contextlib
7
+ import functools
8
+ import inspect
9
+ import os
10
+ from collections.abc import Callable
11
+ from typing import Any
12
+
13
+ import torch
14
+ import triton
15
+
16
+
17
+ FLA_CACHE_RESULTS = os.getenv("FLA_CACHE_RESULTS", "1") == "1"
18
+ SUPPORTS_AUTOTUNE_CACHE = "cache_results" in inspect.signature(triton.autotune).parameters
19
+ autotune_cache_kwargs = (
20
+ {"cache_results": FLA_CACHE_RESULTS} if SUPPORTS_AUTOTUNE_CACHE else {}
21
+ )
22
+
23
+
24
+ @functools.lru_cache(maxsize=1)
25
+ def _backend() -> str:
26
+ try:
27
+ return triton.runtime.driver.active.get_current_target().backend
28
+ except RuntimeError:
29
+ return "cpu"
30
+
31
+
32
+ is_nvidia = _backend() == "cuda"
33
+ is_nvidia_hopper = is_nvidia and (
34
+ "NVIDIA H" in torch.cuda.get_device_name(0)
35
+ or torch.cuda.get_device_capability()[0] >= 9
36
+ )
37
+ is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8
38
+
39
+
40
+ @functools.lru_cache(maxsize=None)
41
+ def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:
42
+ required = {
43
+ "ADA": 101376,
44
+ "AMPERE": 166912,
45
+ "HOPPER": 232448,
46
+ }.get(arch.upper(), 102400)
47
+ try:
48
+ properties = triton.runtime.driver.active.utils.get_device_properties(tensor_idx)
49
+ return properties["max_shared_mem"] >= required
50
+ except Exception:
51
+ return False
52
+
53
+
54
+ def tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]:
55
+ """Retain the source's four-entry cache keyed by input object identity."""
56
+ cache_entries: list[tuple[tuple[Any, ...], dict[str, Any], Any]] = []
57
+ cache_size = 4
58
+
59
+ @functools.wraps(fn)
60
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
61
+ nonlocal cache_entries, cache_size
62
+ for i, entry in enumerate(cache_entries):
63
+ last_args, last_kwargs, last_result = entry
64
+ if len(args) == len(last_args) and len(kwargs) == len(last_kwargs):
65
+ if all(a is b for a, b in zip(args, last_args)) and all(
66
+ k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items()
67
+ ):
68
+ cache_entries = (
69
+ cache_entries[:i]
70
+ + cache_entries[i + 1 :]
71
+ + [(args, kwargs, last_result)]
72
+ )
73
+ return last_result
74
+ result = fn(*args, **kwargs)
75
+ if len(cache_entries) >= cache_size:
76
+ cache_entries = cache_entries[1:]
77
+ cache_entries.append((args, kwargs, result))
78
+ return result
79
+
80
+ return wrapper
81
+
82
+
83
+ def input_guard(fn: Callable[..., Any]) -> Callable[..., Any]:
84
+ """Make tensor inputs contiguous and retain their CUDA device context."""
85
+
86
+ @functools.wraps(fn)
87
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
88
+ contiguous_args = (
89
+ value.contiguous() if isinstance(value, torch.Tensor) else value
90
+ for value in args
91
+ )
92
+ contiguous_kwargs = {
93
+ key: value.contiguous() if isinstance(value, torch.Tensor) else value
94
+ for key, value in kwargs.items()
95
+ }
96
+ tensor = next((value for value in args if isinstance(value, torch.Tensor)), None)
97
+ if tensor is None:
98
+ tensor = next(
99
+ (value for value in kwargs.values() if isinstance(value, torch.Tensor)),
100
+ None,
101
+ )
102
+ context = (
103
+ torch.cuda.device(tensor.device.index)
104
+ if tensor is not None and tensor.is_cuda
105
+ else contextlib.nullcontext()
106
+ )
107
+ with context:
108
+ return fn(*contiguous_args, **contiguous_kwargs)
109
+
110
+ return wrapper
clairic_runtime/gdn/wy_fast.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/wy_fast.py
2
+ # -*- coding: utf-8 -*-
3
+ # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
4
+ # Standalone import layout; see LICENSES and NOTICE for retained notices.
5
+
6
+ from typing import Optional, Tuple
7
+
8
+ import torch
9
+ import triton
10
+ import triton.language as tl
11
+
12
+ from .index import prepare_chunk_indices
13
+
14
+
15
+ # @triton.autotune(
16
+ # configs=[
17
+ # triton.Config({}, num_warps=num_warps, num_stages=num_stages)
18
+ # for num_warps in [2, 4, 8]
19
+ # for num_stages in [2, 3, 4]
20
+ # ],
21
+ # key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
22
+ # )
23
+ @triton.jit(do_not_specialize=["T"])
24
+ def recompute_w_u_fwd_kernel(
25
+ k: tl.tensor,
26
+ v: tl.tensor,
27
+ beta: tl.tensor,
28
+ w: tl.tensor,
29
+ u: tl.tensor,
30
+ A: tl.tensor,
31
+ g: tl.tensor,
32
+ cu_seqlens: tl.tensor,
33
+ chunk_indices: tl.tensor,
34
+ T: tl.int32,
35
+ H: tl.constexpr,
36
+ Hg: tl.constexpr,
37
+ K: tl.constexpr,
38
+ V: tl.constexpr,
39
+ BT: tl.constexpr,
40
+ BK: tl.constexpr,
41
+ BV: tl.constexpr,
42
+ IS_VARLEN: tl.constexpr,
43
+ ) -> None:
44
+ i_t, i_bh = tl.program_id(0), tl.program_id(1)
45
+ i_b, i_h = i_bh // H, i_bh % H
46
+ if IS_VARLEN:
47
+ i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
48
+ chunk_indices + i_t * 2 + 1
49
+ ).to(tl.int32)
50
+ bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
51
+ cu_seqlens + i_n + 1
52
+ ).to(tl.int32)
53
+ T = eos - bos
54
+ else:
55
+ bos, eos = i_b * T, i_b * T + T
56
+ p_beta = tl.make_block_ptr(
57
+ beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
58
+ )
59
+ p_g = tl.make_block_ptr(g + (bos * H + i_h), (T,), (H,), (i_t * BT,), (BT,), (0,))
60
+ p_A = tl.make_block_ptr(
61
+ A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
62
+ )
63
+ b_beta = tl.load(p_beta, boundary_check=(0,))
64
+ b_A = tl.load(p_A, boundary_check=(0, 1))
65
+ b_g = tl.exp(tl.load(p_g, boundary_check=(0,)))
66
+
67
+ for i_v in range(tl.cdiv(V, BV)):
68
+ p_v = tl.make_block_ptr(
69
+ v + (bos * H + i_h) * V,
70
+ (T, V),
71
+ (H * V, 1),
72
+ (i_t * BT, i_v * BV),
73
+ (BT, BV),
74
+ (1, 0),
75
+ )
76
+ p_u = tl.make_block_ptr(
77
+ u + (bos * H + i_h) * V,
78
+ (T, V),
79
+ (H * V, 1),
80
+ (i_t * BT, i_v * BV),
81
+ (BT, BV),
82
+ (1, 0),
83
+ )
84
+ b_v = tl.load(p_v, boundary_check=(0, 1))
85
+ b_vb = (b_v * b_beta[:, None]).to(b_v.dtype)
86
+ b_u = tl.dot(b_A, b_vb, allow_tf32=False)
87
+ tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
88
+
89
+ for i_k in range(tl.cdiv(K, BK)):
90
+ p_k = tl.make_block_ptr(
91
+ k + (bos * Hg + i_h // (H // Hg)) * K,
92
+ (T, K),
93
+ (Hg * K, 1),
94
+ (i_t * BT, i_k * BK),
95
+ (BT, BK),
96
+ (1, 0),
97
+ )
98
+ p_w = tl.make_block_ptr(
99
+ w + (bos * H + i_h) * K,
100
+ (T, K),
101
+ (H * K, 1),
102
+ (i_t * BT, i_k * BK),
103
+ (BT, BK),
104
+ (1, 0),
105
+ )
106
+ b_k = tl.load(p_k, boundary_check=(0, 1))
107
+ b_kb = (b_k * b_beta[:, None] * b_g[:, None]).to(b_k.dtype)
108
+ b_w = tl.dot(b_A, b_kb)
109
+ tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
110
+
111
+
112
+ def recompute_w_u_fwd(
113
+ k: torch.Tensor,
114
+ v: torch.Tensor,
115
+ beta: torch.Tensor,
116
+ g_cumsum: torch.Tensor,
117
+ A: torch.Tensor,
118
+ cu_seqlens: Optional[torch.LongTensor],
119
+ chunk_indices: torch.LongTensor | None = None,
120
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
121
+ B, T, Hg, K, V = *k.shape, v.shape[-1]
122
+ H = v.shape[-2]
123
+ BT = A.shape[-1]
124
+
125
+ if chunk_indices is None and cu_seqlens is not None:
126
+ chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
127
+ NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
128
+ BK = 64
129
+ BV = 64
130
+ u = torch.empty_like(v)
131
+ w = k.new_empty(B, T, H, K)
132
+ recompute_w_u_fwd_kernel[(NT, B * H)](
133
+ k=k,
134
+ v=v,
135
+ beta=beta,
136
+ w=w,
137
+ u=u,
138
+ A=A,
139
+ g=g_cumsum,
140
+ cu_seqlens=cu_seqlens,
141
+ chunk_indices=chunk_indices,
142
+ T=T,
143
+ H=H,
144
+ Hg=Hg,
145
+ K=K,
146
+ V=V,
147
+ BT=BT,
148
+ BK=BK,
149
+ BV=BV,
150
+ IS_VARLEN=cu_seqlens is not None,
151
+ num_warps=4,
152
+ num_stages=3,
153
+ )
154
+ return w, u
155
+
156
+
157
+ fwd_recompute_w_u = recompute_w_u_fwd
clairic_runtime/gdn_pool.py ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Engine-owned native linear-cache storage; dense caches remain fresh native."""
2
+ from __future__ import annotations
3
+
4
+ from typing import Any
5
+
6
+ import torch
7
+
8
+
9
+ class NativeStatePool:
10
+ """One maxB allocation per linear layer, attached to fresh DynamicCache objects.
11
+
12
+ The caller serializes requests on its model worker and must not reuse a cache
13
+ after its request ends. The initialized flags prevent native prefill from
14
+ allocating duplicate state; has_previous_state=False retains original prefill.
15
+ """
16
+
17
+ def __init__(self, backbone: Any, max_batch: int) -> None:
18
+ if max_batch < 1:
19
+ raise ValueError("Positive maximum batch required")
20
+ self.max_batch, self.closed = max_batch, False
21
+ self.language = backbone.model.language_model
22
+ self.states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
23
+ for index, layer in enumerate(self.language.layers):
24
+ if layer.layer_type != "linear_attention":
25
+ continue
26
+ gdn = layer.linear_attn
27
+ weight = gdn.in_proj_qkv.weight
28
+ conv = torch.zeros(max_batch, gdn.conv_dim, gdn.conv_kernel_size,
29
+ dtype=weight.dtype, device=weight.device)
30
+ source = torch.zeros(max_batch, gdn.num_v_heads, gdn.head_v_dim, gdn.head_k_dim,
31
+ dtype=torch.float32, device=weight.device)
32
+ self.states[index] = (conv, source.transpose(-1, -2))
33
+ if not self.states:
34
+ raise ValueError("Expected native linear-attention layers")
35
+
36
+ def attach(self, cache: Any, batch: int, *, previous: bool = False) -> Any:
37
+ if self.closed or not 1 <= batch <= self.max_batch:
38
+ raise RuntimeError("Closed pool or batch outside owned state")
39
+ if len(cache.layers) != len(self.language.layers):
40
+ raise ValueError("Cache layer count differs from language model")
41
+ for index, (conv, recurrent) in self.states.items():
42
+ layer = cache.layers[index]
43
+ # Attach to empty native linear layers only; replacing live storage
44
+ # would silently discard state or retain an unaccounted allocation.
45
+ if layer.is_conv_states_initialized or layer.is_recurrent_states_initialized:
46
+ raise ValueError("Fresh uninitialized native cache required")
47
+ layer.conv_states = conv[:batch]
48
+ layer.recurrent_states = recurrent[:batch]
49
+ layer.is_conv_states_initialized = True
50
+ layer.is_recurrent_states_initialized = True
51
+ layer.has_previous_state = previous
52
+ layer.dtype, layer.device = conv.dtype, conv.device
53
+ layer.max_batch_size, layer.conv_kernel_size = batch, conv.shape[-1]
54
+ return cache
55
+
56
+ def new_cache(self, batch: int, *, previous: bool = False) -> Any:
57
+ if self.closed:
58
+ raise RuntimeError("Closed pool")
59
+ from transformers.cache_utils import DynamicCache
60
+
61
+ return self.attach(DynamicCache(config=self.language.config), batch, previous=previous)
62
+
63
+ def owns(self, cache: Any, batch: int) -> bool:
64
+ if self.closed or not 1 <= batch <= self.max_batch or cache is None:
65
+ return False
66
+ if len(cache.layers) != len(self.language.layers):
67
+ return False
68
+ for index, (conv, recurrent) in self.states.items():
69
+ layer = cache.layers[index]
70
+ for actual, owned in ((layer.conv_states, conv), (layer.recurrent_states, recurrent)):
71
+ if (actual is None or actual.data_ptr() != owned.data_ptr()
72
+ or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
73
+ or actual.dtype != owned.dtype or actual.device != owned.device
74
+ or actual.storage_offset() != owned.storage_offset()
75
+ or tuple(actual.shape) != (batch, *owned.shape[1:])
76
+ or tuple(actual.stride()) != tuple(owned.stride())):
77
+ return False
78
+ if not (layer.is_conv_states_initialized and layer.is_recurrent_states_initialized):
79
+ return False
80
+ return True
81
+
82
+ @property
83
+ def storage_bytes(self) -> int:
84
+ return sum(value.untyped_storage().nbytes() for states in self.states.values() for value in states)
85
+
86
+ def zero(self) -> None:
87
+ if self.closed:
88
+ raise RuntimeError("Closed pool")
89
+ for conv, recurrent in self.states.values():
90
+ conv.zero_()
91
+ recurrent.zero_()
92
+
93
+ def close(self) -> None:
94
+ self.states.clear()
95
+ self.language = None
96
+ self.closed = True
clairic_runtime/kernels.py CHANGED
@@ -172,32 +172,179 @@ def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor,
172
  gate.view(batch, length, q_heads * dim))
173
 
174
 
175
- @triton.jit(do_not_specialize=["XS", "ZS", "YS"])
176
- def _gated_norm_kernel(X: tl.tensor, Z: tl.tensor, W: tl.tensor, Y: tl.tensor,
177
- M: tl.int32, N: tl.constexpr, XS: tl.int64, ZS: tl.int64,
178
- YS: tl.int64, EPS: tl.float32, BLOCK: tl.constexpr) -> None:
179
- rows = tl.program_id(0) * 4 + tl.arange(0, 4)
180
- cols = tl.arange(0, BLOCK)
181
- mask = (rows[:, None] < M) & (cols[None, :] < N)
182
- x = tl.load(X + rows[:, None] * XS + cols[None, :], mask, other=0).to(tl.float32)
183
- xbar = tl.where(mask, x, 0.0)
184
- var = tl.sum(xbar * xbar, axis=1) / N
185
- rstd = tl.rsqrt(var + EPS)
186
- w = tl.load(W + cols, cols < N, other=0).to(tl.float32)
187
- x_hat = x * rstd[:, None]
188
- y = x_hat * w[None, :]
189
- z = tl.load(Z + rows[:, None] * ZS + cols[None, :], mask, other=0).to(tl.float32)
190
- y *= z * tl.sigmoid(z)
191
- tl.store(Y + rows[:, None] * YS + cols[None, :], y, mask)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
 
193
 
194
  def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
195
  eps: float) -> torch.Tensor:
196
  output = torch.empty_like(x)
197
- _gated_norm_kernel[(triton.cdiv(x.shape[0], 4),)](
198
- x, z, weight, output, x.shape[0], x.shape[1], x.stride(0),
199
- z.stride(0), output.stride(0), eps, BLOCK=triton.next_power_of_2(x.shape[1]),
200
- num_warps=min(max(triton.next_power_of_2(x.shape[1]) // 256, 1), 8),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  )
202
  return output
203
 
@@ -476,3 +623,60 @@ def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor |
476
  return c
477
  else:
478
  raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
172
  gate.view(batch, length, q_heads * dim))
173
 
174
 
175
+ @triton.jit
176
+ def _layer_norm_fwd_1pass_kernel(
177
+ X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor,
178
+ Mean: tl.tensor, Rstd: tl.tensor,
179
+ stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64,
180
+ M: tl.int32, N: tl.constexpr, eps: tl.float32,
181
+ BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr,
182
+ HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr,
183
+ IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr,
184
+ USE_GDC: tl.constexpr = False,
185
+ ) -> None:
186
+ if USE_GDC:
187
+ tl.extra.cuda.gdc_wait()
188
+ row_start = tl.program_id(0) * ROWS_PER_BLOCK
189
+ group = tl.program_id(1)
190
+ rows = row_start + tl.arange(0, ROWS_PER_BLOCK)
191
+ cols = tl.arange(0, BLOCK_N)
192
+ row_offsets = rows[:, None] * stride_x_row
193
+ col_offsets = cols[None, :] + group * N
194
+ X_base = X + row_offsets + col_offsets
195
+ Y_base = Y + rows[:, None] * stride_y_row + col_offsets
196
+ row_mask = rows[:, None] < M
197
+ col_mask = cols[None, :] < N
198
+ mask = row_mask & col_mask
199
+ x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32)
200
+ if HAS_Z and (not NORM_BEFORE_GATE):
201
+ Z_base = Z + rows[:, None] * stride_z_row + col_offsets
202
+ z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
203
+ if ACTIVATION == 'swish' or ACTIVATION == 'silu':
204
+ x *= z * tl.sigmoid(z)
205
+ elif ACTIVATION == 'sigmoid':
206
+ x *= tl.sigmoid(z)
207
+ if not IS_RMS_NORM:
208
+ mean = tl.sum(x, axis=1) / N
209
+ mean_offsets = group * M + rows
210
+ mean_mask = rows < M
211
+ tl.store(Mean + mean_offsets, mean, mask=mean_mask)
212
+ xbar = tl.where(mask, x - mean[:, None], 0.0)
213
+ var = tl.sum(xbar * xbar, axis=1) / N
214
+ else:
215
+ xbar = tl.where(mask, x, 0.0)
216
+ var = tl.sum(xbar * xbar, axis=1) / N
217
+ mean = 0.0
218
+ rstd = tl.rsqrt(var + eps)
219
+ rstd_offsets = group * M + rows
220
+ rstd_mask = rows < M
221
+ tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask)
222
+ w_offsets = cols + group * N
223
+ w_mask = cols < N
224
+ w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
225
+ if HAS_BIAS:
226
+ b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
227
+ if not IS_RMS_NORM:
228
+ x_hat = (x - mean[:, None]) * rstd[:, None]
229
+ else:
230
+ x_hat = x * rstd[:, None]
231
+ y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :]
232
+ if HAS_Z and NORM_BEFORE_GATE:
233
+ Z_base = Z + rows[:, None] * stride_z_row + col_offsets
234
+ z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
235
+ if ACTIVATION == 'swish' or ACTIVATION == 'silu':
236
+ y *= z * tl.sigmoid(z)
237
+ elif ACTIVATION == 'sigmoid':
238
+ y *= tl.sigmoid(z)
239
+ tl.store(Y_base, y, mask=mask)
240
+ if USE_GDC:
241
+ tl.extra.cuda.gdc_launch_dependents()
242
 
243
 
244
  def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
245
  eps: float) -> torch.Tensor:
246
  output = torch.empty_like(x)
247
+ rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device)
248
+ block_n = triton.next_power_of_2(x.shape[1])
249
+ pdl = torch.cuda.get_device_capability(x.device)[0] >= 9
250
+ launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {}
251
+ _layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)](
252
+ x, output, weight, None, z, None, rstd,
253
+ x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps,
254
+ BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True,
255
+ NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish",
256
+ num_warps=min(max(block_n // 256, 1), 8), **launch,
257
+ )
258
+ return output
259
+
260
+
261
+ @triton.jit
262
+ def _fused_sigmoid_mul_kernel(
263
+ output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor,
264
+ gate_stride_row: tl.int64, gate_stride_head: tl.int64,
265
+ hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr,
266
+ ) -> None:
267
+ """Fuse sigmoid(gate) * attn_output into a single kernel."""
268
+ pid_row = tl.program_id(0).to(tl.int64)
269
+ pid_block = tl.program_id(1)
270
+ offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H)
271
+ mask = offsets < hidden_dim
272
+ head = offsets // HEAD_DIM
273
+ d = offsets - head * HEAD_DIM
274
+ attn_off = pid_row * hidden_dim + offsets
275
+ attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32)
276
+ gate_off = pid_row * gate_stride_row + head * gate_stride_head + d
277
+ g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32)
278
+ result = attn * tl.sigmoid(g)
279
+ tl.store(output_ptr + attn_off, result, mask=mask)
280
+
281
+ def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor:
282
+ """
283
+ Fused sigmoid-mul for attention output gating.
284
+
285
+ Equivalent to: attn_output * sigmoid(gate)
286
+
287
+ The production Qwen3.5 path passes a 3D strided gate. A single hidden-block
288
+ Triton kernel handles both that path and flat contiguous inputs.
289
+
290
+ When inplace=True, writes result back to attn_output and returns it.
291
+
292
+ Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim)
293
+ and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate
294
+ via explicit strides without requiring a contiguous copy.
295
+ """
296
+ if gate.ndim == 3 and attn_output.ndim == 2:
297
+ num_tokens, num_heads, head_dim = gate.shape
298
+ hidden_dim = num_heads * head_dim
299
+ assert attn_output.shape == (num_tokens, hidden_dim)
300
+ gate_stride_row = gate.stride(0)
301
+ gate_stride_head = gate.stride(1)
302
+ else:
303
+ assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape'
304
+ hidden_dim = attn_output.shape[-1]
305
+ num_tokens = attn_output.numel() // hidden_dim
306
+ head_dim = hidden_dim
307
+ gate_stride_row = hidden_dim
308
+ gate_stride_head = hidden_dim
309
+ out = attn_output if inplace else torch.empty_like(attn_output)
310
+ block_h = 1024 if num_tokens < 1024 else 2048
311
+ grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
312
+ _fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4)
313
+ return out
314
+
315
+
316
+ @triton.jit
317
+ def _silu_mul_kernel(
318
+ gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor,
319
+ elements: tl.constexpr, BLOCK: tl.constexpr,
320
+ ) -> None:
321
+ offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
322
+ mask = offsets < elements
323
+ gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
324
+ up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
325
+ # Preserve the qualified CUDA --use_fast_math operations, including FTZ.
326
+ output = tl.inline_asm_elementwise(
327
+ "{ .reg .f32 exponent, denominator, activated; "
328
+ "mul.ftz.f32 exponent, $1, 0fBFB8AA3B; "
329
+ "ex2.approx.ftz.f32 exponent, exponent; "
330
+ "add.ftz.f32 denominator, exponent, 0f3F800000; "
331
+ "div.approx.ftz.f32 activated, $1, denominator; "
332
+ "mul.ftz.f32 $0, $2, activated; }",
333
+ constraints="=f,f,f", args=[gate, up], dtype=tl.float32,
334
+ is_pure=True, pack=1,
335
+ )
336
+ tl.store(output_ptr + offsets, output, mask=mask)
337
+
338
+
339
+ def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
340
+ """Match the qualified fused CUDA SiLU-mul before the BF16 output store."""
341
+ if (gate.shape != up.shape or gate.device != up.device
342
+ or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16
343
+ or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()):
344
+ raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors")
345
+ output = torch.empty_like(gate)
346
+ _silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)](
347
+ gate, up, output, gate.numel(), BLOCK=2048, num_warps=8,
348
  )
349
  return output
350
 
 
623
  return c
624
  else:
625
  raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')
626
+
627
+
628
+
629
+ @triton.jit(do_not_specialize=["count"])
630
+ def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor,
631
+ count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None:
632
+ offsets = tl.program_id(0) * block + tl.arange(0, block)
633
+ current = tl.load(value + offsets, offsets < count, other=0)
634
+ clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL),
635
+ limit, propagate_nan=tl.PropagateNan.ALL)
636
+ upper = clipped.to(tl.bfloat16)
637
+ lower = (current - upper.to(tl.float32)).to(tl.bfloat16)
638
+ lower = tl.where(current == 0, upper, lower)
639
+ tl.store(high + offsets, upper, offsets < count)
640
+ tl.store(low + offsets, lower, offsets < count)
641
+
642
+
643
+
644
+ def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
645
+ if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous():
646
+ raise ValueError("Expected contiguous CUDA FP32 residuals")
647
+ high = torch.empty_like(value, dtype=torch.bfloat16)
648
+ low = torch.empty_like(value, dtype=torch.bfloat16)
649
+ count = value.numel()
650
+ if count:
651
+ _bf16_pair_kernel[(triton.cdiv(count, 1024),)](
652
+ value, high, low, count, torch.finfo(torch.bfloat16).max, 1024,
653
+ num_warps=4, enable_fp_fusion=False,
654
+ )
655
+ return high, low
656
+
657
+
658
+
659
+ @triton.jit(do_not_specialize=["count"])
660
+ def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor,
661
+ count: tl.int32, block: tl.constexpr) -> None:
662
+ offsets = tl.program_id(0) * block + tl.arange(0, block)
663
+ upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32)
664
+ lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32)
665
+ tl.store(output + offsets, upper + lower, offsets < count)
666
+
667
+
668
+
669
+ def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor:
670
+ if (not first.is_cuda or second.device != first.device or second.shape != first.shape
671
+ or first.dtype not in (torch.bfloat16, torch.float32)
672
+ or second.dtype != torch.bfloat16
673
+ or not first.is_contiguous() or not second.is_contiguous()):
674
+ raise ValueError("Expected aligned contiguous CUDA residuals")
675
+ output = torch.empty_like(first, dtype=torch.float32)
676
+ count = first.numel()
677
+ if count:
678
+ _residual_kernel[(triton.cdiv(count, 1024),)](
679
+ first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False,
680
+ )
681
+ return output
682
+
clairic_runtime/math.py CHANGED
@@ -29,10 +29,22 @@ from transformers.models.qwen3_5.modeling_qwen3_5 import (
29
  eager_attention_forward, torch_causal_conv1d_update,
30
  )
31
 
 
 
 
32
 
33
  def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
34
  if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
35
  raise ValueError("Expected aligned matrices with equal dtype")
 
 
 
 
 
 
 
 
 
36
  from .kernels import _matmul_persistent_triton
37
  return _matmul_persistent_triton(a, b, bias)
38
 
@@ -83,15 +95,40 @@ class ReadoutMath(TorchDispatchMode):
83
  return function(*args, **kwargs)
84
 
85
 
86
- def _linear_forward(original: Callable[..., Any], lm_head: bool) -> Callable[..., torch.Tensor]:
 
 
 
 
87
  def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
88
  if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
89
  or input.dtype != self.weight.dtype):
 
90
  return original(input)
91
  shape = input.shape[:-1]
92
  flat = input.reshape(-1, input.shape[-1])
93
  if lm_head:
94
- output = _matmul(flat.contiguous(), self.weight.T.contiguous()).float()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  else:
96
  output = _matmul(flat, self.weight.T, self.bias)
97
  return output.reshape(*shape, self.weight.shape[0])
@@ -99,11 +136,29 @@ def _linear_forward(original: Callable[..., Any], lm_head: bool) -> Callable[...
99
 
100
 
101
  def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
 
 
 
102
  value = x.float()
103
  value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
104
  return (value * (1.0 + self.weight.float())).to(self.weight.dtype)
105
 
106
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
108
  position_embeddings: tuple[torch.Tensor, torch.Tensor],
109
  attention_mask: torch.Tensor | None = None,
@@ -119,13 +174,17 @@ def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
119
  states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
120
  position_ids=position_ids, past_key_values=past_key_values,
121
  position_embeddings=position_embeddings, **kwargs)
122
- merged = residual.float() + states.float()
123
  states = self.post_attention_layernorm(merged)
124
- return merged.to(dtype).float() + self.mlp(states).float()
125
 
126
 
127
  def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
128
  gate, up = self.gate_proj(x), self.up_proj(x)
 
 
 
 
129
  return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))
130
 
131
 
@@ -146,6 +205,7 @@ def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor,
146
  def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
147
  position_embeddings: tuple[torch.Tensor, torch.Tensor],
148
  attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
 
149
  **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
150
  input_shape = hidden_states.shape[:-1]
151
  hidden_shape = (*input_shape, -1, self.head_dim)
@@ -165,11 +225,20 @@ def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
165
  key, value = past_key_values.update(key, value, self.layer_idx)
166
  interface = ALL_ATTENTION_FUNCTIONS.get_interface(
167
  self.config._attn_implementation, eager_attention_forward)
168
- output, weights = interface(self, query, key, value, attention_mask,
169
- dropout=0.0 if not self.training else self.attention_dropout,
170
- scaling=self.scaling, **kwargs)
 
 
 
 
 
171
  output = output.reshape(*input_shape, -1).contiguous()
172
- output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype)
 
 
 
 
173
  return self.o_proj(output), weights
174
 
175
 
@@ -201,27 +270,16 @@ def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor,
201
  mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
202
  if cached:
203
  mixed = mixed[:, :, -length:]
204
- query, key, value = torch.split(mixed.transpose(1, 2),
205
- (self.key_dim, self.key_dim, self.value_dim), -1)
206
- query = query.reshape(batch, length, -1, self.head_k_dim)
207
- key = key.reshape(batch, length, -1, self.head_k_dim)
208
- value = value.reshape(batch, length, -1, self.head_v_dim)
209
- beta = b.float().sigmoid() if cached and length == 1 else b.sigmoid()
210
- g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
211
- if self.num_v_heads // self.num_k_heads > 1:
212
- groups = self.num_v_heads // self.num_k_heads
213
- query, key = query.repeat_interleave(groups, 2), key.repeat_interleave(groups, 2)
214
  if cached and length == 1:
215
- output, state = self.recurrent_gated_delta_rule(query, key, value, g=g, beta=beta,
216
- initial_state=recurrent_state, output_final_state=cache_params is not None,
217
- use_qk_l2norm_in_kernel=True)
218
  else:
219
- output, state = self.chunk_gated_delta_rule(query, key, value, g=g, beta=beta,
220
- initial_state=recurrent_state if cached else None,
221
- output_final_state=cache_params is not None, use_qk_l2norm_in_kernel=True,
222
- cu_seqlens=kwargs.get("cu_seq_lens_q"))
223
  if cache_params is not None:
224
- cache_params.update_recurrent_state(state, self.layer_idx)
225
  output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
226
  return self.out_proj(output.reshape(batch, length, -1))
227
 
@@ -309,6 +367,8 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
309
  forwards: list[tuple[nn.Module, bool, Any, Any]] = []
310
  convolutions: list[tuple[Any, Any, Any]] = []
311
  caches: dict[torch.device, torch.Tensor] = {}
 
 
312
  closed = False
313
 
314
  def restore() -> None:
@@ -316,6 +376,7 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
316
  if closed:
317
  return
318
  closed = True
 
319
  for module, original_prefill, original_update in reversed(convolutions):
320
  module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
321
  for module, had_override, original, installed in reversed(forwards):
@@ -325,6 +386,9 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
325
  else:
326
  del module.forward
327
  caches.clear()
 
 
 
328
 
329
  def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
330
  parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
@@ -364,13 +428,24 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
364
  elif isinstance(module, Qwen3_5MLP):
365
  bind(module, _mlp, "x", module.gate_proj.weight)
366
  elif isinstance(module, Qwen3_5Attention):
367
- bind(module, _dense, parameter=module.q_proj.weight)
 
 
368
  elif isinstance(module, Qwen3_5GatedDeltaNet):
369
- bind(module, _gdn, parameter=module.in_proj_qkv.weight)
370
  bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
371
  convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
372
  module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
373
  bind(rotary, text_rotary, "x")
 
 
 
 
 
 
 
 
 
374
  lm_head = backbone.get_output_embeddings()
375
  roots = (backbone,) if head is None else (backbone, head)
376
  seen: set[int] = set()
@@ -382,7 +457,7 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
382
  if module.weight.dtype not in (torch.bfloat16, torch.float32):
383
  continue
384
  original = module.forward
385
- installed = MethodType(_linear_forward(original, module is lm_head), module)
386
  forwards.append((module, "forward" in module.__dict__, original, installed))
387
  module.forward = installed
388
  except BaseException:
 
29
  eager_attention_forward, torch_causal_conv1d_update,
30
  )
31
 
32
+ from .attention_metadata import DecodeMetadata
33
+ from .recurrent_metadata import scope as recurrent_metadata_scope
34
+
35
 
36
  def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
37
  if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
38
  raise ValueError("Expected aligned matrices with equal dtype")
39
+ if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
40
+ and a.is_contiguous() and b.transpose(0, 1).is_contiguous()
41
+ and b.shape[1] >= 16):
42
+ from deep_gemm import bf16_gemm_nn
43
+ output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype)
44
+ bf16_gemm_nn(a, b, output)
45
+ if bias is not None:
46
+ output += bias
47
+ return output
48
  from .kernels import _matmul_persistent_triton
49
  return _matmul_persistent_triton(a, b, bias)
50
 
 
95
  return function(*args, **kwargs)
96
 
97
 
98
+ def _linear_forward(original: Callable[..., Any], lm_head: bool,
99
+ transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]:
100
+ cache: dict[str, Any] = {}
101
+ if lm_head and transpose_caches is not None:
102
+ transpose_caches.append(cache)
103
  def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
104
  if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
105
  or input.dtype != self.weight.dtype):
106
+ cache.clear()
107
  return original(input)
108
  shape = input.shape[:-1]
109
  flat = input.reshape(-1, input.shape[-1])
110
  if lm_head:
111
+ weight = self.weight
112
+ version = None
113
+ if not torch.is_grad_enabled() and not torch.compiler.is_compiling():
114
+ try:
115
+ version = weight._version
116
+ except RuntimeError:
117
+ pass
118
+ if version is None:
119
+ cache.clear()
120
+ transposed = weight.T.contiguous()
121
+ else:
122
+ stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None
123
+ key = (id(weight), version, weight.data_ptr(), weight.storage_offset(),
124
+ weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream)
125
+ if cache.get("weight") is not weight or cache.get("key") != key:
126
+ cache.clear()
127
+ cache["transposed"] = weight.T.contiguous()
128
+ cache["key"] = key
129
+ cache["weight"] = weight
130
+ transposed = cache["transposed"]
131
+ output = _matmul(flat.contiguous(), transposed).float()
132
  else:
133
  output = _matmul(flat, self.weight.T, self.bias)
134
  return output.reshape(*shape, self.weight.shape[0])
 
136
 
137
 
138
  def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
139
+ if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32):
140
+ from .normalization import rms_norm
141
+ return rms_norm(x, self.weight, self.eps)
142
  value = x.float()
143
  value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
144
  return (value * (1.0 + self.weight.float())).to(self.weight.dtype)
145
 
146
 
147
+ def _residual_sum(first: torch.Tensor, second: torch.Tensor,
148
+ dtype: torch.dtype) -> torch.Tensor:
149
+ if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda
150
+ and first.device == second.device and first.shape == second.shape
151
+ and first.dtype in (torch.bfloat16, torch.float32)
152
+ and second.dtype == torch.bfloat16
153
+ and first.is_contiguous() and second.is_contiguous()
154
+ and first.numel() <= 2**31 - 1
155
+ and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
156
+ from .kernels import residual_sum
157
+ return residual_sum(first, second)
158
+ return first.to(dtype).float() + second.float()
159
+
160
+
161
+
162
  def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
163
  position_embeddings: tuple[torch.Tensor, torch.Tensor],
164
  attention_mask: torch.Tensor | None = None,
 
174
  states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
175
  position_ids=position_ids, past_key_values=past_key_values,
176
  position_embeddings=position_embeddings, **kwargs)
177
+ merged = _residual_sum(residual, states, dtype)
178
  states = self.post_attention_layernorm(merged)
179
+ return _residual_sum(merged, self.mlp(states), dtype)
180
 
181
 
182
  def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
183
  gate, up = self.gate_proj(x), self.up_proj(x)
184
+ if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16
185
+ and gate.is_contiguous() and up.is_contiguous()):
186
+ from .kernels import silu_mul
187
+ return self.down_proj(silu_mul(gate, up))
188
  return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))
189
 
190
 
 
205
  def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
206
  position_embeddings: tuple[torch.Tensor, torch.Tensor],
207
  attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
208
+ *, metadata: DecodeMetadata | None = None,
209
  **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
210
  input_shape = hidden_states.shape[:-1]
211
  hidden_shape = (*input_shape, -1, self.head_dim)
 
225
  key, value = past_key_values.update(key, value, self.layer_idx)
226
  interface = ALL_ATTENTION_FUNCTIONS.get_interface(
227
  self.config._attn_implementation, eager_attention_forward)
228
+ if metadata is not None:
229
+ output, weights = metadata.attend(interface, self, query, key, value, attention_mask,
230
+ dropout=0.0 if not self.training else self.attention_dropout,
231
+ scaling=self.scaling, **kwargs)
232
+ else:
233
+ output, weights = interface(self, query, key, value, attention_mask,
234
+ dropout=0.0 if not self.training else self.attention_dropout,
235
+ scaling=self.scaling, **kwargs)
236
  output = output.reshape(*input_shape, -1).contiguous()
237
+ if output.is_cuda and output.dtype == torch.bfloat16:
238
+ from .kernels import fused_sigmoid_mul
239
+ output = fused_sigmoid_mul(output, gate, inplace=True)
240
+ else:
241
+ output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype)
242
  return self.o_proj(output), weights
243
 
244
 
 
270
  mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
271
  if cached:
272
  mixed = mixed[:, :, -length:]
273
+ from .gdn.adapter import decode, prefill, store_state
 
 
 
 
 
 
 
 
 
274
  if cached and length == 1:
275
+ output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
276
+ recurrent_state)
 
277
  else:
278
+ output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
279
+ self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim,
280
+ recurrent_state if cached else None, attention_mask)
 
281
  if cache_params is not None:
282
+ store_state(cache_params, self.layer_idx, state)
283
  output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
284
  return self.out_proj(output.reshape(batch, length, -1))
285
 
 
367
  forwards: list[tuple[nn.Module, bool, Any, Any]] = []
368
  convolutions: list[tuple[Any, Any, Any]] = []
369
  caches: dict[torch.device, torch.Tensor] = {}
370
+ transpose_caches: list[dict[str, Any]] = []
371
+ attention_metadata = DecodeMetadata()
372
  closed = False
373
 
374
  def restore() -> None:
 
376
  if closed:
377
  return
378
  closed = True
379
+ attention_metadata.close()
380
  for module, original_prefill, original_update in reversed(convolutions):
381
  module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
382
  for module, had_override, original, installed in reversed(forwards):
 
386
  else:
387
  del module.forward
388
  caches.clear()
389
+ for cache in transpose_caches:
390
+ cache.clear()
391
+ transpose_caches.clear()
392
 
393
  def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
394
  parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
 
428
  elif isinstance(module, Qwen3_5MLP):
429
  bind(module, _mlp, "x", module.gate_proj.weight)
430
  elif isinstance(module, Qwen3_5Attention):
431
+ def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any:
432
+ return _dense(self, *args, metadata=attention_metadata, **kwargs)
433
+ bind(module, dense, parameter=module.q_proj.weight)
434
  elif isinstance(module, Qwen3_5GatedDeltaNet):
435
+ bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True)
436
  bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
437
  convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
438
  module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
439
  bind(rotary, text_rotary, "x")
440
+ original_language_forward = language.forward
441
+
442
+ def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
443
+ with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self):
444
+ return original_language_forward(*args, **kwargs)
445
+
446
+ installed_language = MethodType(language_forward, language)
447
+ forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language))
448
+ language.forward = installed_language
449
  lm_head = backbone.get_output_embeddings()
450
  roots = (backbone,) if head is None else (backbone, head)
451
  seen: set[int] = set()
 
457
  if module.weight.dtype not in (torch.bfloat16, torch.float32):
458
  continue
459
  original = module.forward
460
+ installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module)
461
  forwards.append((module, "forward" in module.__dict__, original, installed))
462
  module.forward = installed
463
  except BaseException:
clairic_runtime/native.py CHANGED
@@ -7,8 +7,8 @@ import concurrent.futures
7
  import copy
8
  import json
9
  import threading
10
- from collections.abc import Callable
11
- from contextlib import nullcontext
12
  from collections import defaultdict
13
  from pathlib import Path
14
  from typing import Any
@@ -18,10 +18,52 @@ from . import model as cm
18
  from .readout import load_head, read_record
19
  from .math import ReadoutMath, install as install_math
20
  from .vision_cache import VisionBatchCache
 
21
 
22
  _FLASH_ATTENTION = 'kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0'
23
 
24
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  class _ConstrainedScores:
26
  """Capture probabilities after applying the schema's grammar constraints."""
27
 
@@ -30,12 +72,34 @@ class _ConstrainedScores:
30
  self.score_ids = score_ids
31
  self.chosen: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
32
  self.first: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
 
 
 
33
 
34
  def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor:
 
 
35
  masked = self.processor(input_ids, scores.clone())
36
  logprobs = torch.log_softmax(masked.float(), dim=-1)
37
- if not self.chosen[0]:
38
- selected = logprobs[:, self.score_ids].cpu().tolist()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
  self.first = [
40
  [
41
  (float(value), token, None)
@@ -43,13 +107,9 @@ class _ConstrainedScores:
43
  ]
44
  for row in selected
45
  ]
46
- choices = masked.argmax(dim=-1)
47
- values = logprobs.gather(1, choices[:, None])[:, 0].cpu().tolist()
48
- for row, token, value in zip(
49
- self.chosen, choices.cpu().tolist(), values, strict=True
50
- ):
51
- row.append((float(value), int(token), None))
52
- return masked
53
 
54
 
55
  class _CancellationCriteria:
@@ -76,20 +136,23 @@ class TransformersEngine:
76
  head: cm.ClairicHead,
77
  tokenizer: Any,
78
  *,
79
- batch_size: int = 8,
80
  text_batch_size: int | None = None,
81
- batch_token_budget: int = 8192,
82
  ) -> None:
83
  import xgrammar as xgr
84
 
85
  self.backbone = backbone.eval()
86
  self.head = head.to(backbone.device).float().eval()
87
  self.tokenizer = tokenizer
88
- self.batch_size = max(1, int(batch_size))
 
89
  self.text_batch_size = (
90
  self.batch_size if text_batch_size is None else max(1, int(text_batch_size))
91
  )
92
- self.batch_token_budget = max(1, int(batch_token_budget))
 
 
93
  self.device, self.dtype = backbone.device, backbone.dtype
94
  self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
95
  self.image_token = int(backbone.config.image_token_id)
@@ -106,6 +169,8 @@ class TransformersEngine:
106
  self._uncached_image_features = self.backbone.model.get_image_features
107
  self.backbone.model.get_image_features = self._image_features
108
  self._restore_math: Callable[[], None] | None = None
 
 
109
 
110
  def begin_request(self) -> None:
111
  self._vision_cache.begin()
@@ -137,9 +202,9 @@ class TransformersEngine:
137
  *,
138
  device: str = "cuda",
139
  dtype: Any = None,
140
- batch_size: int = 8,
141
  text_batch_size: int | None = None,
142
- batch_token_budget: int = 8192,
143
  **kwargs: Any,
144
  ) -> TransformersEngine:
145
  from transformers import Qwen3_5ForConditionalGeneration
@@ -152,23 +217,31 @@ class TransformersEngine:
152
  if isinstance(dtype, str):
153
  dtype = getattr(torch, dtype)
154
  kwargs.setdefault("device_map", {"": device})
 
155
  if "attn_implementation" not in kwargs:
156
  attention = "sdpa"
157
  if str(device).startswith("cuda") and dtype == torch.bfloat16:
158
  if torch.cuda.get_device_capability(torch.device(device))[0] == 9:
159
  attention = _FLASH_ATTENTION
 
160
  kwargs["attn_implementation"] = {
161
  "text_config": attention, "vision_config": attention,
162
  }
163
- backbone = _NativeQwen3_5ForConditionalGeneration.from_pretrained(
164
- path,
165
- dtype=dtype,
166
- local_files_only=True,
167
- **kwargs,
168
- )
 
169
  engine = cls(
170
  backbone,
171
- load_head(path),
 
 
 
 
 
172
  tokenizer,
173
  batch_size=batch_size,
174
  text_batch_size=text_batch_size,
@@ -176,6 +249,9 @@ class TransformersEngine:
176
  )
177
  try:
178
  engine._restore_math = install_math(backbone, engine.head)
 
 
 
179
  except BaseException:
180
  engine.shutdown()
181
  raise
@@ -307,16 +383,21 @@ class TransformersEngine:
307
  config.max_new_tokens, config.use_cache = maximum, True
308
  config.eos_token_id = list(stop_ids)
309
  config.pad_token_id = self.tokenizer.pad_token_id or self.eos
310
- generated = self.backbone.generate(
311
- **inputs,
312
- generation_config=config,
313
- logits_processor=[scores],
314
- stopping_criteria=[_CancellationCriteria(_cancelled)]
315
- if _cancelled is not None
316
- else [],
317
- )
 
 
 
 
318
  if _cancelled is not None and _cancelled.is_set():
319
  raise RuntimeError("Inference cancelled")
 
320
  continuations = (
321
  generated[:, inputs["input_ids"].shape[1] :].cpu().tolist()
322
  )
@@ -429,9 +510,27 @@ class TransformersEngine:
429
  lambda cancelled: self._read_typed(item, media, _cancelled=cancelled)
430
  )
431
 
 
 
 
 
 
432
  def shutdown(self) -> None:
433
  try:
434
- self._worker.shutdown(wait=True, cancel_futures=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
435
  self._vision_cache.clear()
436
  if self.backbone.model.get_image_features == self._image_features:
437
  self.backbone.model.get_image_features = self._uncached_image_features
 
7
  import copy
8
  import json
9
  import threading
10
+ from collections.abc import Callable, Iterator
11
+ from contextlib import contextmanager, nullcontext
12
  from collections import defaultdict
13
  from pathlib import Path
14
  from typing import Any
 
18
  from .readout import load_head, read_record
19
  from .math import ReadoutMath, install as install_math
20
  from .vision_cache import VisionBatchCache
21
+ from .decoder_graph import prepare as prepare_decoder_graphs
22
 
23
  _FLASH_ATTENTION = 'kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0'
24
 
25
 
26
+ def _batch_defaults(backbone: Any) -> tuple[int, int]:
27
+ config = getattr(backbone.config, "text_config", None)
28
+ if (backbone.device.type == "cuda" and backbone.dtype == torch.bfloat16
29
+ and config is not None and config._attn_implementation == _FLASH_ATTENTION):
30
+ if config.hidden_size == 4096:
31
+ return 64, 65536
32
+ if config.hidden_size == 5120:
33
+ return 32, 32768
34
+ return 8, 8192
35
+
36
+
37
+ _ATTENTION_KERNEL_LOCK = threading.RLock()
38
+
39
+
40
+ @contextmanager
41
+ def _default_attention_kernel() -> Iterator[None]:
42
+ """Select the pinned public attention build while initializing its loader."""
43
+ import os
44
+ from huggingface_hub import snapshot_download
45
+ from transformers.modeling_flash_attention_utils import lazy_import_flash_attention
46
+
47
+ repository, revision = _FLASH_ATTENTION.rsplit("@", 1)
48
+ build_name = "torch-stable-abi210-cu130-x86_64-linux"
49
+ snapshot = snapshot_download(repository, revision=revision,
50
+ allow_patterns=[f"build/{build_name}/**"])
51
+ build = Path(snapshot) / "build" / build_name
52
+ with _ATTENTION_KERNEL_LOCK:
53
+ previous = os.environ.get("LOCAL_KERNELS")
54
+ entries = previous.split(":") if previous else []
55
+ entries = [entry for entry in entries if entry.partition("=")[0] != repository]
56
+ os.environ["LOCAL_KERNELS"] = ":".join([*entries, f"{repository}={build}"])
57
+ try:
58
+ yield
59
+ lazy_import_flash_attention(implementation=_FLASH_ATTENTION)
60
+ finally:
61
+ if previous is None:
62
+ os.environ.pop("LOCAL_KERNELS", None)
63
+ else:
64
+ os.environ["LOCAL_KERNELS"] = previous
65
+
66
+
67
  class _ConstrainedScores:
68
  """Capture probabilities after applying the schema's grammar constraints."""
69
 
 
72
  self.score_ids = score_ids
73
  self.chosen: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
74
  self.first: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
75
+ self._tokens: list[torch.Tensor] = []
76
+ self._logprobs: list[torch.Tensor] = []
77
+ self._first_values: torch.Tensor | None = None
78
 
79
  def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor:
80
+ if self.processor.prefilled and input_ids.is_cuda:
81
+ input_ids = input_ids[:, -1:].cpu()
82
  masked = self.processor(input_ids, scores.clone())
83
  logprobs = torch.log_softmax(masked.float(), dim=-1)
84
+ if not self._tokens:
85
+ self._first_values = logprobs[:, self.score_ids]
86
+ choices = masked.argmax(dim=-1)
87
+ self._tokens.append(choices)
88
+ self._logprobs.append(logprobs.gather(1, choices[:, None])[:, 0])
89
+ return masked
90
+
91
+ def finalize(self) -> None:
92
+ if not self._tokens:
93
+ return
94
+ tokens = torch.stack(self._tokens, dim=1).cpu().tolist()
95
+ values = torch.stack(self._logprobs, dim=1).cpu().tolist()
96
+ self.chosen = [
97
+ [(float(value), int(token), None)
98
+ for token, value in zip(row_tokens, row_values, strict=True)]
99
+ for row_tokens, row_values in zip(tokens, values, strict=True)
100
+ ]
101
+ if self._first_values is not None:
102
+ selected = self._first_values.cpu().tolist()
103
  self.first = [
104
  [
105
  (float(value), token, None)
 
107
  ]
108
  for row in selected
109
  ]
110
+ self._tokens.clear()
111
+ self._logprobs.clear()
112
+ self._first_values = None
 
 
 
 
113
 
114
 
115
  class _CancellationCriteria:
 
136
  head: cm.ClairicHead,
137
  tokenizer: Any,
138
  *,
139
+ batch_size: int | None = None,
140
  text_batch_size: int | None = None,
141
+ batch_token_budget: int | None = None,
142
  ) -> None:
143
  import xgrammar as xgr
144
 
145
  self.backbone = backbone.eval()
146
  self.head = head.to(backbone.device).float().eval()
147
  self.tokenizer = tokenizer
148
+ default_size, default_budget = _batch_defaults(backbone)
149
+ self.batch_size = max(1, int(default_size if batch_size is None else batch_size))
150
  self.text_batch_size = (
151
  self.batch_size if text_batch_size is None else max(1, int(text_batch_size))
152
  )
153
+ self.batch_token_budget = max(1, int(
154
+ default_budget if batch_token_budget is None else batch_token_budget
155
+ ))
156
  self.device, self.dtype = backbone.device, backbone.dtype
157
  self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
158
  self.image_token = int(backbone.config.image_token_id)
 
169
  self._uncached_image_features = self.backbone.model.get_image_features
170
  self.backbone.model.get_image_features = self._image_features
171
  self._restore_math: Callable[[], None] | None = None
172
+ self._decoder_graphs: Any = None
173
+ self._decoder_cleanup: Any = None
174
 
175
  def begin_request(self) -> None:
176
  self._vision_cache.begin()
 
202
  *,
203
  device: str = "cuda",
204
  dtype: Any = None,
205
+ batch_size: int | None = None,
206
  text_batch_size: int | None = None,
207
+ batch_token_budget: int | None = None,
208
  **kwargs: Any,
209
  ) -> TransformersEngine:
210
  from transformers import Qwen3_5ForConditionalGeneration
 
217
  if isinstance(dtype, str):
218
  dtype = getattr(torch, dtype)
219
  kwargs.setdefault("device_map", {"": device})
220
+ default_flash = False
221
  if "attn_implementation" not in kwargs:
222
  attention = "sdpa"
223
  if str(device).startswith("cuda") and dtype == torch.bfloat16:
224
  if torch.cuda.get_device_capability(torch.device(device))[0] == 9:
225
  attention = _FLASH_ATTENTION
226
+ default_flash = True
227
  kwargs["attn_implementation"] = {
228
  "text_config": attention, "vision_config": attention,
229
  }
230
+ with _default_attention_kernel() if default_flash else nullcontext():
231
+ backbone = _NativeQwen3_5ForConditionalGeneration.from_pretrained(
232
+ path,
233
+ dtype=dtype,
234
+ local_files_only=True,
235
+ **kwargs,
236
+ )
237
  engine = cls(
238
  backbone,
239
+ load_head(
240
+ path,
241
+ match_bfloat16=(
242
+ backbone.device.type == "cuda" and backbone.dtype == torch.bfloat16
243
+ ),
244
+ ),
245
  tokenizer,
246
  batch_size=batch_size,
247
  text_batch_size=text_batch_size,
 
249
  )
250
  try:
251
  engine._restore_math = install_math(backbone, engine.head)
252
+ engine._decoder_graphs = engine._worker.submit(
253
+ prepare_decoder_graphs, backbone, max(engine.batch_size, engine.text_batch_size)
254
+ ).result()
255
  except BaseException:
256
  engine.shutdown()
257
  raise
 
383
  config.max_new_tokens, config.use_cache = maximum, True
384
  config.eos_token_id = list(stop_ids)
385
  config.pad_token_id = self.tokenizer.pad_token_id or self.eos
386
+ scope = (self._decoder_graphs.request(len(chunk))
387
+ if self._decoder_graphs is not None else nullcontext(None))
388
+ with scope as native_cache:
389
+ generated = self.backbone.generate(
390
+ **inputs,
391
+ **({"past_key_values": native_cache} if native_cache is not None else {}),
392
+ generation_config=config,
393
+ logits_processor=[scores],
394
+ stopping_criteria=[_CancellationCriteria(_cancelled)]
395
+ if _cancelled is not None
396
+ else [],
397
+ )
398
  if _cancelled is not None and _cancelled.is_set():
399
  raise RuntimeError("Inference cancelled")
400
+ scores.finalize()
401
  continuations = (
402
  generated[:, inputs["input_ids"].shape[1] :].cpu().tolist()
403
  )
 
510
  lambda cancelled: self._read_typed(item, media, _cancelled=cancelled)
511
  )
512
 
513
+ def _decoder_diagnostics(self) -> Any:
514
+ if self._decoder_graphs is not None:
515
+ return self._decoder_graphs.diagnostics()
516
+ return self._decoder_cleanup
517
+
518
  def shutdown(self) -> None:
519
  try:
520
+ manager, self._decoder_graphs = self._decoder_graphs, None
521
+ def close_graphs() -> None:
522
+ try:
523
+ manager.close()
524
+ finally:
525
+ self._decoder_cleanup = manager.diagnostics()
526
+ try:
527
+ if manager is not None:
528
+ if threading.get_ident() == manager.thread:
529
+ close_graphs()
530
+ else:
531
+ self._worker.submit(close_graphs).result()
532
+ finally:
533
+ self._worker.shutdown(wait=True, cancel_futures=True)
534
  self._vision_cache.clear()
535
  if self.backbone.model.get_image_features == self._image_features:
536
  self.backbone.model.get_image_features = self._uncached_image_features
clairic_runtime/normalization.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Public CUDA normalization with the model's residual precision."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+
8
+ def _bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
9
+ """Recover BF16 operands for an FP32 sum of two BF16 residuals."""
10
+ if (value.is_cuda and value.dtype == torch.float32 and value.is_contiguous()
11
+ and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
12
+ from .kernels import bf16_pair_sum
13
+ return bf16_pair_sum(value)
14
+ limit = torch.finfo(torch.bfloat16).max
15
+ high = value.clamp(-limit, limit).to(torch.bfloat16)
16
+ low = (value - high.float()).to(torch.bfloat16)
17
+ # Equal signed zeros keep the sign when the fused operation adds them.
18
+ low = torch.where(value == 0, high, low)
19
+ return high, low
20
+
21
+
22
+ def rms_norm(value: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
23
+ """Normalize CUDA BF16 states or unrounded FP32 residual sums."""
24
+ from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
25
+
26
+ shape = value.shape
27
+ flat = value.reshape(-1, shape[-1]).contiguous()
28
+ if value.dtype == torch.bfloat16:
29
+ return gemma_rmsnorm(flat, weight, eps).reshape(shape)
30
+ if value.dtype != torch.float32:
31
+ raise ValueError("Expected BF16 states or FP32 residual sums")
32
+ high, low = _bf16_pair_sum(flat)
33
+ gemma_fused_add_rmsnorm(low, high, weight, eps)
34
+ return low.reshape(shape)
clairic_runtime/readout.py CHANGED
@@ -12,13 +12,23 @@ from safetensors.torch import load_file
12
  from . import model as cm
13
 
14
 
15
- def load_head(path: Path) -> cm.ClairicHead:
16
  configuration = json.loads((path / "clair_head_config.json").read_text())
17
  accepted = inspect.signature(cm.ClairicHead).parameters
18
  head = cm.ClairicHead(
19
  **{name: value for name, value in configuration.items() if name in accepted}
20
  )
21
- head.load_state_dict(load_file(path / "clair_head.safetensors"), strict=True)
 
 
 
 
 
 
 
 
 
 
22
  return head.float().eval()
23
 
24
 
 
12
  from . import model as cm
13
 
14
 
15
+ def load_head(path: Path, *, match_bfloat16: bool = False) -> cm.ClairicHead:
16
  configuration = json.loads((path / "clair_head_config.json").read_text())
17
  accepted = inspect.signature(cm.ClairicHead).parameters
18
  head = cm.ClairicHead(
19
  **{name: value for name, value in configuration.items() if name in accepted}
20
  )
21
+ state = load_file(path / "clair_head.safetensors")
22
+ if match_bfloat16:
23
+ # Serving constructs the head in BF16 before widening it to FP32.
24
+ # The length bias has an explicit FP32 initialization.
25
+ state = {
26
+ name: value.bfloat16().float()
27
+ if value.is_floating_point() and name != "length_bias"
28
+ else value
29
+ for name, value in state.items()
30
+ }
31
+ head.load_state_dict(state, strict=True)
32
  return head.float().eval()
33
 
34
 
clairic_runtime/recurrent_metadata.py ADDED
@@ -0,0 +1,131 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Share read-only recurrent packing metadata during one inference forward."""
2
+ from __future__ import annotations
3
+
4
+ from collections.abc import Callable, Iterator
5
+ from contextlib import contextmanager
6
+ from contextvars import ContextVar
7
+ from typing import Any
8
+
9
+ import torch
10
+
11
+
12
+ class _Forward:
13
+ def __init__(self, hook_owner: torch.nn.Module | None = None) -> None:
14
+ self.hook_owner = hook_owner
15
+ self.unversioned_mask_allowed: bool | None = None
16
+ self.hook_checks = 0
17
+ self.key: tuple[Any, ...] | None = None
18
+ self.mask: torch.Tensor | None = None
19
+ self.packed: tuple[torch.Tensor, torch.Tensor] | None = None
20
+ self.slots_key: tuple[Any, ...] | None = None
21
+ self.slots: torch.Tensor | None = None
22
+ self.materializations = 0
23
+ self.hits = 0
24
+ self.slots_materializations = 0
25
+ self.slots_hits = 0
26
+
27
+ def clear(self) -> None:
28
+ self.key, self.mask, self.packed = None, None, None
29
+ self.slots_key, self.slots = None, None
30
+ self.hook_owner, self.unversioned_mask_allowed = None, None
31
+
32
+
33
+ _current: ContextVar[_Forward | None] = ContextVar("clairic_recurrent_metadata", default=None)
34
+ _MAX_BYTES = 8 * 1024**2
35
+
36
+
37
+ @contextmanager
38
+ def scope(hook_owner: torch.nn.Module | None = None) -> Iterator[None]:
39
+ """Drop all added references on success, cancellation, or model failure."""
40
+ state = _Forward(hook_owner)
41
+ token = _current.set(state)
42
+ try:
43
+ yield
44
+ finally:
45
+ state.clear()
46
+ _current.reset(token)
47
+
48
+
49
+ def _eligible(mixed: torch.Tensor) -> _Forward | None:
50
+ state = _current.get()
51
+ if (state is None or not mixed.is_cuda or mixed.dtype != torch.bfloat16
52
+ or mixed.ndim != 3 or torch.is_grad_enabled()
53
+ or not torch.is_inference_mode_enabled() or torch.compiler.is_compiling()):
54
+ return None
55
+ return state
56
+
57
+
58
+ def _no_forward_hooks(owner: torch.nn.Module | None) -> bool:
59
+ """Admit only a known hook-free tree and hook-free global registries."""
60
+ if not isinstance(owner, torch.nn.Module):
61
+ return False
62
+ registry = torch.nn.modules.module
63
+ for name in ("_global_forward_pre_hooks", "_global_forward_hooks"):
64
+ hooks = getattr(registry, name, None)
65
+ if hooks is None or hooks:
66
+ return False
67
+ for module in owner.modules():
68
+ for name in ("_forward_pre_hooks", "_forward_hooks"):
69
+ hooks = getattr(module, name, None)
70
+ if hooks is None or hooks:
71
+ return False
72
+ return True
73
+
74
+
75
+ def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None,
76
+ original: Callable[[torch.Tensor, torch.Tensor | None],
77
+ tuple[torch.Tensor, torch.Tensor]]
78
+ ) -> tuple[torch.Tensor, torch.Tensor]:
79
+ """Reuse the original helper's exact results for an unchanged current mask."""
80
+ state = _eligible(mixed)
81
+ batch, length = mixed.shape[:2]
82
+ required = batch * length * 8 + (2 * batch + 1) * 4
83
+ if state is None or required > _MAX_BYTES:
84
+ return original(mixed, mask)
85
+ stream = torch.cuda.current_stream(mixed.device).cuda_stream
86
+ mask_key: tuple[Any, ...] = ()
87
+ if mask is not None:
88
+ try:
89
+ version = mask._version
90
+ except RuntimeError:
91
+ version = None
92
+ if version is None:
93
+ if state.unversioned_mask_allowed is None:
94
+ state.hook_checks += 1
95
+ state.unversioned_mask_allowed = _no_forward_hooks(state.hook_owner)
96
+ if not state.unversioned_mask_allowed:
97
+ state.key, state.mask, state.packed = None, None, None
98
+ return original(mixed, mask)
99
+ mask_key = (id(mask), version, mask.data_ptr(), mask.storage_offset(),
100
+ mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride()))
101
+ key = (batch, length, mixed.device, stream, mask_key)
102
+ if state.key == key and state.mask is mask and state.packed is not None:
103
+ state.hits += 1
104
+ return state.packed
105
+ state.key, state.mask, state.packed = None, None, None
106
+ if state.slots_key != (batch, mixed.device, stream):
107
+ state.slots_key, state.slots = None, None
108
+ packed = original(mixed, mask)
109
+ retained = sum(value.numel() * value.element_size() for value in packed)
110
+ if retained + batch * 4 <= _MAX_BYTES:
111
+ state.key, state.mask, state.packed = key, mask, packed
112
+ state.materializations += 1
113
+ return packed
114
+
115
+
116
+ def sequence_slots(mixed: torch.Tensor) -> torch.Tensor:
117
+ """Reuse the same identity slot vector in each recurrent layer."""
118
+ state = _eligible(mixed)
119
+ batch = mixed.shape[0]
120
+ retained = (sum(value.numel() * value.element_size() for value in state.packed)
121
+ if state is not None and state.packed is not None else 0)
122
+ if state is None or retained + batch * 4 > _MAX_BYTES:
123
+ return torch.arange(batch, dtype=torch.int32, device=mixed.device)
124
+ key = (batch, mixed.device, torch.cuda.current_stream(mixed.device).cuda_stream)
125
+ if state.slots_key != key or state.slots is None:
126
+ state.slots = torch.arange(batch, dtype=torch.int32, device=mixed.device)
127
+ state.slots_key = key
128
+ state.slots_materializations += 1
129
+ else:
130
+ state.slots_hits += 1
131
+ return state.slots
clairic_runtime/row_omission.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Recover unresolved image rows backed by a completed row inventory."""
2
+
3
+ from copy import deepcopy
4
+ import json
5
+ from typing import Any
6
+
7
+ from . import ar, image_rows
8
+ from .schema import MAX_LIST_ITEMS
9
+
10
+
11
+ def prepare(
12
+ record: dict[str, Any], predictions: dict[str, dict[str, Any]]
13
+ ) -> tuple[dict[str, Any], list[dict[str, Any]]] | None:
14
+ if record.get("_null_recovery") or record.get("_row_omission_recovery"):
15
+ return None
16
+ annotations = (
17
+ record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
18
+ )
19
+ tasks = {name: (name, ids, media) for name, ids, media in record["_ar_tasks"]}
20
+ if len(tasks) != len(record["_ar_tasks"]):
21
+ return None
22
+ contexts, retry_tasks = [], []
23
+ constraints = dict(record.get("_ar_constraints", {}))
24
+ for parent, annotation in annotations.items():
25
+ queries = annotation.get("image_row_fields")
26
+ spec, inventory = record["fields"].get(parent, {}), predictions.get(parent, {})
27
+ value = inventory.get("value")
28
+ if (
29
+ set(annotation) != {"image_row_fields"}
30
+ or not isinstance(queries, list)
31
+ or len(queries) != len(set(queries))
32
+ or not queries
33
+ or not image_rows.is_inventory(spec)
34
+ or spec.get("knowledge")
35
+ or not inventory.get("valid")
36
+ or not inventory.get("complete")
37
+ or not isinstance(value, list)
38
+ or not value
39
+ or len(value) > min(spec.get("max_items", MAX_LIST_ITEMS), len(queries))
40
+ or any(
41
+ not isinstance(row, dict)
42
+ or set(row) != {"label"}
43
+ or not isinstance(row["label"], str)
44
+ or not row["label"].strip()
45
+ for row in value
46
+ )
47
+ or any(child not in predictions for child in queries)
48
+ ):
49
+ continue
50
+ properties = spec.get("items", {}).get("properties", {})
51
+ text_keys = [
52
+ key
53
+ for key, child in properties.items()
54
+ if child.get("type") == "text" and not child.get("knowledge")
55
+ ]
56
+ if not text_keys:
57
+ continue
58
+ labels = [row["label"] for row in value]
59
+ proposals = [predictions[child] for child in queries]
60
+ selected, conflicts = image_rows._select(labels, proposals, text_keys)
61
+ if not conflicts or any(
62
+ row is not None and (not row.get("valid") or not row.get("complete"))
63
+ for row in selected
64
+ ):
65
+ continue
66
+ children, local_constraints = [], {}
67
+ for index in conflicts:
68
+ child = queries[index]
69
+ row, child_spec = predictions[child], record["fields"].get(child, {})
70
+ row_value = row.get("value")
71
+ task = tasks.get(child)
72
+ native = constraints.get(child, {})
73
+ if (
74
+ annotations.get(child) != {"parent": parent, "index": index + 1}
75
+ or child_spec.get("type") != "object"
76
+ or not child_spec.get("_image_row_proposal")
77
+ or child_spec.get("knowledge")
78
+ or not row.get("valid")
79
+ or not row.get("complete")
80
+ or not isinstance(row_value, dict)
81
+ or not row_value
82
+ or any(cell is not None for cell in row_value.values())
83
+ or any(row is previous for previous in selected if previous is not None)
84
+ or task is None
85
+ or task[2] is None
86
+ or set(native) != {"json_schema"}
87
+ ):
88
+ break
89
+ schema = json.loads(native["json_schema"])
90
+ if schema != ar.schema(child_spec) or not set(text_keys) <= set(
91
+ schema.get("properties", {})
92
+ ):
93
+ break
94
+ # A completed inventory requires a matching textual cell; the
95
+ # original selector remains the acceptance test for that label.
96
+ # Full branches retain required fields with the native grammar compiler.
97
+ branches = []
98
+ for key in text_keys:
99
+ branch = deepcopy(schema)
100
+ branch["properties"][key]["type"] = "string"
101
+ branches.append(branch)
102
+ local_constraints[child] = {"json_schema": json.dumps({"anyOf": branches})}
103
+ children.append(child)
104
+ else:
105
+ constraints.update(local_constraints)
106
+ retry_tasks.extend(tasks[child] for child in children)
107
+ contexts.append(
108
+ {
109
+ "parent": parent,
110
+ "queries": queries,
111
+ "labels": labels,
112
+ "text_keys": text_keys,
113
+ "selected": selected,
114
+ "conflicts": conflicts,
115
+ "children": children,
116
+ }
117
+ )
118
+ if not retry_tasks:
119
+ return None
120
+ repair = {
121
+ **record,
122
+ "_ar_tasks": retry_tasks,
123
+ "_ar_constraints": constraints,
124
+ "_ar_prefix": None,
125
+ "_ar_group_prefixes": [],
126
+ "_ar_media_groups": [],
127
+ "_null_recovery": True,
128
+ "_row_omission_recovery": True,
129
+ }
130
+ return repair, contexts
131
+
132
+
133
+ def merge(
134
+ record: dict[str, Any],
135
+ predictions: dict[str, dict[str, Any]],
136
+ alternatives: dict[str, dict[str, Any]],
137
+ plan: tuple[dict[str, Any], list[dict[str, Any]]],
138
+ ) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]:
139
+ repair, contexts = plan
140
+ lengths = {name: len(ids) for name, ids, _ in repair["_ar_tasks"]}
141
+ result = dict(predictions)
142
+ evidence = []
143
+ for context in contexts:
144
+ candidates = {}
145
+ for child in context["children"]:
146
+ original, alternative = predictions[child], alternatives.get(child, {})
147
+ account = {
148
+ "tokens": original["tokens"] + alternative.get("tokens", 0),
149
+ "retry_input_tokens": original.get("retry_input_tokens", 0)
150
+ + lengths[child],
151
+ }
152
+ result[child] = {**original, **account}
153
+ value = alternative.get("value")
154
+ if (
155
+ alternative.get("valid")
156
+ and alternative.get("complete")
157
+ and isinstance(value, dict)
158
+ and any(cell is not None for cell in value.values())
159
+ ):
160
+ candidates[child] = {
161
+ **alternative,
162
+ **account,
163
+ "image_row_omission_recovered": True,
164
+ }
165
+ trial = {**result, **candidates}
166
+ selected, conflicts = image_rows._select(
167
+ context["labels"],
168
+ [trial[name] for name in context["queries"]],
169
+ context["text_keys"],
170
+ )
171
+ unchanged = all(
172
+ previous is None or selected[index] is previous
173
+ for index, previous in enumerate(context["selected"])
174
+ )
175
+ accepted = (
176
+ not conflicts and unchanged and len(candidates) == len(context["children"])
177
+ )
178
+ if accepted:
179
+ result.update(candidates)
180
+ evidence.append(
181
+ {
182
+ "parent": context["parent"],
183
+ "original_conflicts": context["conflicts"],
184
+ "retry_children": context["children"],
185
+ "conflicts_after_retry": conflicts,
186
+ "previously_matched_proposals_preserved_exactly": unchanged,
187
+ "accepted": accepted,
188
+ }
189
+ )
190
+ return result, evidence
requirements.txt CHANGED
@@ -1,9 +1,15 @@
1
- torch==2.11.0
2
- torchvision==0.26.0
3
  transformers==5.10.2
4
  kernels==0.13.0
5
  tomlkit==0.13.3
6
- triton==3.6.0; sys_platform == "linux" and platform_machine == "x86_64"
 
 
 
 
 
 
7
  flash-linear-attention==0.6.0
8
  fla-core==0.6.0
9
  einops==0.8.1
 
1
+ torch==2.13.0
2
+ torchvision==0.28.0
3
  transformers==5.10.2
4
  kernels==0.13.0
5
  tomlkit==0.13.3
6
+ sgl-deep-gemm==0.1.5.post3
7
+ apache-tvm-ffi==0.1.11
8
+ flashinfer-python==0.6.17; sys_platform == "linux" and platform_machine == "x86_64"
9
+ flashinfer-cubin @ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.17/flashinfer_cubin-0.6.17-py3-none-any.whl#sha256=771f037a828cdf3f15db1c3784574197546d1b09735fb9058f384bdd12010afa ; sys_platform == "linux" and platform_machine == "x86_64"
10
+ flashinfer-jit-cache @ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.17/flashinfer_jit_cache-0.6.17+cu130-cp39-abi3-manylinux_2_28_x86_64.whl#sha256=e7dd1deecc7f348d41b1eb56217d945d4f1630f447ff9b0b9e7919b0e07c01f0 ; sys_platform == "linux" and platform_machine == "x86_64"
11
+ nvidia-cutlass-dsl[cu13]==4.6.2; sys_platform == "linux" and platform_machine == "x86_64"
12
+ triton==3.7.1; sys_platform == "linux" and platform_machine == "x86_64"
13
  flash-linear-attention==0.6.0
14
  fla-core==0.6.0
15
  einops==0.8.1