Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Update Clairic Transformers runtime
Browse files- README.md +2 -2
- clairic_runtime/attention_metadata.py +156 -0
- clairic_runtime/backend.py +5 -0
- clairic_runtime/decoder_capture.py +220 -0
- clairic_runtime/decoder_graph.py +289 -0
- clairic_runtime/gdn/LICENSES +229 -0
- clairic_runtime/gdn/NOTICE +14 -0
- clairic_runtime/gdn/__init__.py +1 -0
- clairic_runtime/gdn/adapter.py +112 -0
- clairic_runtime/gdn/chunk_delta_h.py +386 -0
- clairic_runtime/gdn/chunk_fwd.py +417 -0
- clairic_runtime/gdn/chunk_o.py +175 -0
- clairic_runtime/gdn/cumsum.py +106 -0
- clairic_runtime/gdn/gating.py +77 -0
- clairic_runtime/gdn/index.py +36 -0
- clairic_runtime/gdn/l2norm.py +101 -0
- clairic_runtime/gdn/op.py +28 -0
- clairic_runtime/gdn/prefill.py +62 -0
- clairic_runtime/gdn/recurrent.py +226 -0
- clairic_runtime/gdn/utils.py +110 -0
- clairic_runtime/gdn/wy_fast.py +157 -0
- clairic_runtime/gdn_pool.py +96 -0
- clairic_runtime/kernels.py +225 -21
- clairic_runtime/math.py +104 -29
- clairic_runtime/native.py +132 -33
- clairic_runtime/normalization.py +34 -0
- clairic_runtime/readout.py +12 -2
- clairic_runtime/recurrent_metadata.py +131 -0
- clairic_runtime/row_omission.py +190 -0
- requirements.txt +9 -3
README.md
CHANGED
|
@@ -35,9 +35,9 @@ Use Python 3.11 or later and install the packages listed in `requirements.txt`.
|
|
| 35 |
Transformers; `xgrammar` supplies schema constraints through a Transformers logits
|
| 36 |
processor.
|
| 37 |
|
| 38 |
-
The supported GPU setup is Linux x86_64 with an H100 or H200, PyTorch 2.
|
| 39 |
with CUDA 13.0, and Transformers 5.10.2. The CUDA BF16 default uses FlashAttention 3
|
| 40 |
-
for text and vision, with batches of up to
|
| 41 |
the pinned attention kernel through `kernels`; cache it before running offline.
|
| 42 |
Explicit `attn_implementation`, `batch_size`, and `text_batch_size` loader options
|
| 43 |
remain available.
|
|
|
|
| 35 |
Transformers; `xgrammar` supplies schema constraints through a Transformers logits
|
| 36 |
processor.
|
| 37 |
|
| 38 |
+
The supported GPU setup is Linux x86_64 with an H100 or H200, PyTorch 2.13.0
|
| 39 |
with CUDA 13.0, and Transformers 5.10.2. The CUDA BF16 default uses FlashAttention 3
|
| 40 |
+
for text and vision, with batches of up to 32 sequences. The first load fetches
|
| 41 |
the pinned attention kernel through `kernels`; cache it before running offline.
|
| 42 |
Explicit `attn_implementation`, `batch_size`, and `text_batch_size` loader options
|
| 43 |
remain available.
|
clairic_runtime/attention_metadata.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reuse pinned FA3 unpadding metadata within one owned decode forward."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from collections.abc import Callable, Iterator
|
| 5 |
+
from contextlib import contextmanager
|
| 6 |
+
from contextvars import ContextVar
|
| 7 |
+
import threading
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from transformers.integrations.flash_attention import flash_attention_forward
|
| 12 |
+
from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis
|
| 13 |
+
|
| 14 |
+
_PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0"
|
| 15 |
+
_VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"})
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class _Entry:
|
| 19 |
+
def __init__(self) -> None:
|
| 20 |
+
self.key: tuple[Any, ...] | None = None
|
| 21 |
+
self.mask: torch.Tensor | None = None
|
| 22 |
+
self.indices: torch.Tensor | None = None
|
| 23 |
+
self.cu_k: torch.Tensor | None = None
|
| 24 |
+
self.cu_q: torch.Tensor | None = None
|
| 25 |
+
self.maximum = 0
|
| 26 |
+
self.bytes = 0
|
| 27 |
+
|
| 28 |
+
def clear(self) -> None:
|
| 29 |
+
self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None
|
| 30 |
+
self.maximum, self.bytes = 0, 0
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class DecodeMetadata:
|
| 34 |
+
def __init__(self, max_bytes: int = 8 * 1024**2) -> None:
|
| 35 |
+
if max_bytes <= 0:
|
| 36 |
+
raise ValueError("Expected a positive metadata bound")
|
| 37 |
+
self.max_bytes = max_bytes
|
| 38 |
+
self._lock = threading.Lock()
|
| 39 |
+
self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None)
|
| 40 |
+
self._active: _Entry | None = None
|
| 41 |
+
self._closed = False
|
| 42 |
+
self.scopes = 0
|
| 43 |
+
self.materializations = 0
|
| 44 |
+
self.hits = 0
|
| 45 |
+
self.peak_bytes = 0
|
| 46 |
+
|
| 47 |
+
@contextmanager
|
| 48 |
+
def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]:
|
| 49 |
+
token = self._current.set(None)
|
| 50 |
+
acquired = False
|
| 51 |
+
entry = None
|
| 52 |
+
try:
|
| 53 |
+
inputs = kwargs.get("input_ids", args[0] if args else None)
|
| 54 |
+
if inputs is None:
|
| 55 |
+
inputs = kwargs.get("inputs_embeds")
|
| 56 |
+
eligible = (not self._closed and not torch.is_grad_enabled()
|
| 57 |
+
and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling()
|
| 58 |
+
and isinstance(inputs, torch.Tensor) and inputs.is_cuda
|
| 59 |
+
and inputs.ndim in (2, 3) and inputs.shape[1] == 1)
|
| 60 |
+
if eligible and self._lock.acquire(blocking=False):
|
| 61 |
+
acquired = True
|
| 62 |
+
if not self._closed:
|
| 63 |
+
entry = _Entry()
|
| 64 |
+
self._active = entry
|
| 65 |
+
self._current.set(entry)
|
| 66 |
+
self.scopes += 1
|
| 67 |
+
yield
|
| 68 |
+
finally:
|
| 69 |
+
if entry is not None:
|
| 70 |
+
entry.clear()
|
| 71 |
+
self._active = None
|
| 72 |
+
self._current.reset(token)
|
| 73 |
+
if acquired:
|
| 74 |
+
self._lock.release()
|
| 75 |
+
|
| 76 |
+
def _metadata(self, mask: torch.Tensor) -> _Entry | None:
|
| 77 |
+
entry = self._current.get()
|
| 78 |
+
if self._closed or entry is None:
|
| 79 |
+
return None
|
| 80 |
+
batch = mask.shape[0]
|
| 81 |
+
if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes:
|
| 82 |
+
entry.clear()
|
| 83 |
+
return None
|
| 84 |
+
try:
|
| 85 |
+
version = mask._version
|
| 86 |
+
except RuntimeError:
|
| 87 |
+
version = None
|
| 88 |
+
key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device,
|
| 89 |
+
mask.dtype, tuple(mask.shape), tuple(mask.stride()),
|
| 90 |
+
torch.cuda.current_stream(mask.device).cuda_stream)
|
| 91 |
+
if entry.mask is mask and entry.key == key:
|
| 92 |
+
self.hits += 1
|
| 93 |
+
return entry
|
| 94 |
+
entry.clear()
|
| 95 |
+
indices, cu_k, maximum = _get_unpad_data(mask)
|
| 96 |
+
cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device)
|
| 97 |
+
retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q))
|
| 98 |
+
if retained > self.max_bytes:
|
| 99 |
+
return None
|
| 100 |
+
entry.key, entry.mask = key, mask
|
| 101 |
+
entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q
|
| 102 |
+
entry.maximum, entry.bytes = int(maximum.item()), retained
|
| 103 |
+
self.materializations += 1
|
| 104 |
+
self.peak_bytes = max(self.peak_bytes, retained)
|
| 105 |
+
return entry
|
| 106 |
+
|
| 107 |
+
def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]],
|
| 108 |
+
module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
|
| 109 |
+
mask: torch.Tensor | None, *, dropout: float, scaling: float,
|
| 110 |
+
**kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
|
| 111 |
+
eligible = (not self._closed and self._current.get() is not None
|
| 112 |
+
and not torch.is_grad_enabled() and torch.is_inference_mode_enabled()
|
| 113 |
+
and not torch.compiler.is_compiling() and interface is flash_attention_forward
|
| 114 |
+
and module.config._attn_implementation == _PINNED_FA3
|
| 115 |
+
and dropout == 0.0
|
| 116 |
+
and getattr(module.config, "sliding_window", None) is None
|
| 117 |
+
and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys())
|
| 118 |
+
and mask is not None and mask.is_cuda and mask.ndim == 2
|
| 119 |
+
and mask.dtype == torch.bool and mask.numel() > 0
|
| 120 |
+
and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4
|
| 121 |
+
and item.device == mask.device and item.stride(-1) == 1
|
| 122 |
+
for item in (query, key, value)))
|
| 123 |
+
if eligible:
|
| 124 |
+
assert mask is not None
|
| 125 |
+
positions = kwargs.get("position_ids")
|
| 126 |
+
eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0]
|
| 127 |
+
and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1]
|
| 128 |
+
and mask.shape[1] > 1
|
| 129 |
+
and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4
|
| 130 |
+
and query.shape[3] == key.shape[3] == value.shape[3] == 256
|
| 131 |
+
and (positions is None or (isinstance(positions, torch.Tensor)
|
| 132 |
+
and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1))))
|
| 133 |
+
entry = self._metadata(mask) if eligible and mask is not None else None
|
| 134 |
+
if entry is None:
|
| 135 |
+
active = self._current.get()
|
| 136 |
+
if active is not None:
|
| 137 |
+
active.clear()
|
| 138 |
+
return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs)
|
| 139 |
+
assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None
|
| 140 |
+
packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
|
| 141 |
+
packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
|
| 142 |
+
return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling,
|
| 143 |
+
cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k,
|
| 144 |
+
max_length_q=1, max_length_k=entry.maximum, **kwargs)
|
| 145 |
+
|
| 146 |
+
def snapshot(self) -> dict[str, int | bool]:
|
| 147 |
+
return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits,
|
| 148 |
+
"peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0,
|
| 149 |
+
"closed": self._closed}
|
| 150 |
+
|
| 151 |
+
def close(self) -> None:
|
| 152 |
+
self._closed = True
|
| 153 |
+
with self._lock:
|
| 154 |
+
if self._active is not None:
|
| 155 |
+
self._active.clear()
|
| 156 |
+
self._active = None
|
clairic_runtime/backend.py
CHANGED
|
@@ -40,6 +40,7 @@ from . import (
|
|
| 40 |
printed_numbers,
|
| 41 |
printed_temporal,
|
| 42 |
receipt_totals,
|
|
|
|
| 43 |
source_dates,
|
| 44 |
source_rows,
|
| 45 |
table_rows,
|
|
@@ -1113,6 +1114,10 @@ class ClairicFlashModel:
|
|
| 1113 |
predictions[name] = {**alternative, "null_recovered": True}
|
| 1114 |
predictions[name]["tokens"] = tokens
|
| 1115 |
predictions[name]["retry_input_tokens"] = retry_lengths[name]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1116 |
return predictions
|
| 1117 |
|
| 1118 |
def null_recovery_record(
|
|
|
|
| 40 |
printed_numbers,
|
| 41 |
printed_temporal,
|
| 42 |
receipt_totals,
|
| 43 |
+
row_omission,
|
| 44 |
source_dates,
|
| 45 |
source_rows,
|
| 46 |
table_rows,
|
|
|
|
| 1114 |
predictions[name] = {**alternative, "null_recovered": True}
|
| 1115 |
predictions[name]["tokens"] = tokens
|
| 1116 |
predictions[name]["retry_input_tokens"] = retry_lengths[name]
|
| 1117 |
+
if plan := row_omission.prepare(record, predictions):
|
| 1118 |
+
repair, _ = plan
|
| 1119 |
+
alternatives = await self._ar_all(repair, eos)
|
| 1120 |
+
predictions, _ = row_omission.merge(record, predictions, alternatives, plan)
|
| 1121 |
return predictions
|
| 1122 |
|
| 1123 |
def null_recovery_record(
|
clairic_runtime/decoder_capture.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Owned single-layer graph capture and strict resource lifecycle."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
import threading
|
| 5 |
+
from typing import Any
|
| 6 |
+
import weakref
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def tensor_key(value: Any, *, version: bool) -> tuple[Any, ...]:
|
| 10 |
+
try:
|
| 11 |
+
counter = value._version if version else None
|
| 12 |
+
except RuntimeError:
|
| 13 |
+
counter = None
|
| 14 |
+
storage = value.untyped_storage()
|
| 15 |
+
return (id(value), counter, value.data_ptr(), storage.data_ptr(), storage.nbytes(),
|
| 16 |
+
value.storage_offset(), str(value.device), str(value.dtype),
|
| 17 |
+
tuple(value.shape), tuple(value.stride()))
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def callable_key(value: Any) -> tuple[int, int | None]:
|
| 21 |
+
return (id(getattr(value, "__func__", value)),
|
| 22 |
+
id(value.__self__) if hasattr(value, "__self__") else None)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def input_key(value: Any) -> tuple[Any, ...]:
|
| 26 |
+
return str(value.device), str(value.dtype), tuple(value.shape), tuple(value.stride())
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def owner_key(torch: Any, owner: Any, cache: Any, operation: Any) -> tuple[Any, ...]:
|
| 30 |
+
modules = []
|
| 31 |
+
for name, module in owner.named_modules():
|
| 32 |
+
if module._forward_hooks or module._forward_pre_hooks:
|
| 33 |
+
raise RuntimeError("Forward hooks invalidate decoder capture")
|
| 34 |
+
scalars = tuple(sorted((key, value) for key, value in vars(module).items()
|
| 35 |
+
if not key.startswith("_") and isinstance(value, (str, bool, int, float))))
|
| 36 |
+
callables = tuple(sorted((key, callable_key(value)) for key, value in vars(module).items()
|
| 37 |
+
if not key.startswith("_") and callable(value)))
|
| 38 |
+
modules.append((name, id(module), callable_key(module.forward), scalars, callables))
|
| 39 |
+
registry = torch.nn.modules.module
|
| 40 |
+
if registry._global_forward_hooks or registry._global_forward_pre_hooks:
|
| 41 |
+
raise RuntimeError("Global forward hooks invalidate decoder capture")
|
| 42 |
+
parameters = tuple((name, tensor_key(value, version=True))
|
| 43 |
+
for name, value in owner.named_parameters())
|
| 44 |
+
if any(item[1][1] is None for item in parameters):
|
| 45 |
+
raise RuntimeError("Versioned parameters required")
|
| 46 |
+
buffers = tuple((name, tensor_key(value, version=True))
|
| 47 |
+
for name, value in owner.named_buffers())
|
| 48 |
+
layer = cache.layers[0]
|
| 49 |
+
if not (cache.has_previous_state(0) and layer.is_conv_states_initialized
|
| 50 |
+
and layer.is_recurrent_states_initialized):
|
| 51 |
+
raise RuntimeError("Already initialized live decode cache required")
|
| 52 |
+
states = tuple(tensor_key(value, version=False)
|
| 53 |
+
for value in (layer.conv_states, layer.recurrent_states))
|
| 54 |
+
return (id(owner), id(cache), id(layer), callable_key(operation), tuple(modules),
|
| 55 |
+
parameters, buffers, states, callable_key(cache.update_recurrent_state),
|
| 56 |
+
callable_key(cache.has_previous_state), callable_key(layer.update_recurrent_state))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def describe_pool(segments: list[dict[str, Any]], pool: tuple[int, int]) -> dict[str, Any]:
|
| 60 |
+
blocks = [block for segment in segments for block in segment["blocks"]]
|
| 61 |
+
pending = {"active_awaiting_free", "active_pending_free"}
|
| 62 |
+
return {"pool": pool, "segments": segments,
|
| 63 |
+
"foreign_pool": any(tuple(segment["segment_pool_id"]) != pool for segment in segments),
|
| 64 |
+
"reserved_bytes": sum(segment["total_size"] for segment in segments),
|
| 65 |
+
"allocated_bytes": sum(segment["allocated_size"] for segment in segments),
|
| 66 |
+
"active_bytes": sum(segment["active_size"] for segment in segments),
|
| 67 |
+
"active_allocated_blocks": sum(block["state"] == "active_allocated" for block in blocks),
|
| 68 |
+
"pending_blocks": sum(block["state"] in pending for block in blocks),
|
| 69 |
+
"unknown_block_states": sorted({block["state"] for block in blocks}
|
| 70 |
+
- pending - {"active_allocated", "inactive"})}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class DecoderGraph:
|
| 74 |
+
"""One exact bucket/dtype graph with owned input and cloned FP32 outputs.
|
| 75 |
+
|
| 76 |
+
Capture/replay must run on the engine's exclusive CUDA worker. Mutation is
|
| 77 |
+
intentional only for the two captured cache tensors. Any owner, parameter,
|
| 78 |
+
callable, stream, runtime, cache storage or hook change rejects the session.
|
| 79 |
+
Capture/replay failures propagate; callers must close the engine rather than
|
| 80 |
+
retrying eager work on a possibly invalid CUDA context.
|
| 81 |
+
"""
|
| 82 |
+
|
| 83 |
+
def __init__(self, backend: Any, owner: Any, cache: Any, value: Any,
|
| 84 |
+
operation: Any, *, exclusive_process: bool) -> None:
|
| 85 |
+
if not exclusive_process:
|
| 86 |
+
raise ValueError("Exclusive process required")
|
| 87 |
+
self.backend, self.owner, self.cache, self.operation = backend, owner, cache, operation
|
| 88 |
+
self.thread, self.closed = threading.get_ident(), False
|
| 89 |
+
self.context = backend.context(value)
|
| 90 |
+
self.input = input_key(value)
|
| 91 |
+
self.key = owner_key(backend.torch, owner, cache, operation)
|
| 92 |
+
self.entry = backend.capture(value, operation, cache)
|
| 93 |
+
self.check(value)
|
| 94 |
+
|
| 95 |
+
def check(self, value: Any) -> None:
|
| 96 |
+
if self.closed or self.thread != threading.get_ident():
|
| 97 |
+
raise RuntimeError("Closed graph or foreign thread")
|
| 98 |
+
if self.backend.context(value) != self.context or input_key(value) != self.input:
|
| 99 |
+
raise RuntimeError("Input layout, dtype, stream or runtime changed")
|
| 100 |
+
if owner_key(self.backend.torch, self.owner, self.cache, self.operation) != self.key:
|
| 101 |
+
raise RuntimeError("Decoder owner, parameter, callable or cache changed")
|
| 102 |
+
|
| 103 |
+
def run(self, value: Any) -> Any:
|
| 104 |
+
self.check(value)
|
| 105 |
+
return self.direct(value)
|
| 106 |
+
|
| 107 |
+
def direct(self, value: Any) -> Any:
|
| 108 |
+
# The padded manager executes complete guards before direct replay.
|
| 109 |
+
return self.backend.replay_clone(self.entry, value)
|
| 110 |
+
|
| 111 |
+
def close(self) -> dict[str, Any]:
|
| 112 |
+
if self.closed or self.thread != threading.get_ident():
|
| 113 |
+
raise RuntimeError("Closed graph or foreign thread")
|
| 114 |
+
details = self.backend.release(self.entry)
|
| 115 |
+
self.owner = self.cache = self.operation = self.key = None
|
| 116 |
+
self.closed = True
|
| 117 |
+
return details
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
class TorchBackend:
|
| 121 |
+
def __init__(self, torch: Any, runtime_key: Any) -> None:
|
| 122 |
+
self.torch, self.runtime_key = torch, runtime_key
|
| 123 |
+
|
| 124 |
+
def context(self, value: Any) -> tuple[Any, ...]:
|
| 125 |
+
torch = self.torch
|
| 126 |
+
if (torch.is_grad_enabled() or torch.compiler.is_compiling() or not value.is_cuda
|
| 127 |
+
or torch.cuda.is_current_stream_capturing() or not value.is_contiguous()
|
| 128 |
+
or value.dtype not in (torch.bfloat16, torch.float32)
|
| 129 |
+
or value.ndim != 3 or value.shape[1] != 1 or min(value.shape) < 1):
|
| 130 |
+
raise RuntimeError("Unsupported decoder graph context/input")
|
| 131 |
+
return (torch.cuda.current_stream(value.device).cuda_stream, self.runtime_key())
|
| 132 |
+
|
| 133 |
+
def capture(self, value: Any, operation: Any, cache: Any) -> dict[str, Any]:
|
| 134 |
+
torch = self.torch
|
| 135 |
+
current = torch.cuda.current_stream(value.device)
|
| 136 |
+
side = torch.cuda.Stream(device=value.device)
|
| 137 |
+
side.wait_stream(current)
|
| 138 |
+
layer = cache.layers[0]
|
| 139 |
+
states = (layer.conv_states, layer.recurrent_states)
|
| 140 |
+
saved = tuple(state.clone(memory_format=torch.preserve_format) for state in states)
|
| 141 |
+
side.wait_stream(current)
|
| 142 |
+
|
| 143 |
+
def restore(current_states: Any, before_states: Any) -> None:
|
| 144 |
+
for state, before in zip(current_states, before_states, strict=True):
|
| 145 |
+
state.copy_(before)
|
| 146 |
+
|
| 147 |
+
graph = torch.cuda.CUDAGraph()
|
| 148 |
+
with torch.cuda.stream(side):
|
| 149 |
+
static_input = torch.empty_like(value)
|
| 150 |
+
static_input.copy_(value)
|
| 151 |
+
for _ in range(2):
|
| 152 |
+
warm_output = operation(static_input)
|
| 153 |
+
side.synchronize()
|
| 154 |
+
del warm_output
|
| 155 |
+
restore(states, saved)
|
| 156 |
+
side.synchronize()
|
| 157 |
+
graph.capture_begin(capture_error_mode="global")
|
| 158 |
+
try:
|
| 159 |
+
static_output = operation(static_input)
|
| 160 |
+
finally:
|
| 161 |
+
graph.capture_end()
|
| 162 |
+
side.synchronize()
|
| 163 |
+
restore(states, saved)
|
| 164 |
+
current.wait_stream(side)
|
| 165 |
+
current.synchronize()
|
| 166 |
+
assert static_output.dtype == torch.float32
|
| 167 |
+
static_input.record_stream(current)
|
| 168 |
+
static_output.record_stream(current)
|
| 169 |
+
for state, before in zip(states, saved, strict=True):
|
| 170 |
+
assert torch.equal(state.contiguous().view(torch.int16), before.contiguous().view(torch.int16))
|
| 171 |
+
del saved, states
|
| 172 |
+
pool = tuple(graph.pool())
|
| 173 |
+
details = describe_pool(torch.cuda.memory_snapshot(mempool_id=pool, include_traces=False), pool)
|
| 174 |
+
assert not details["foreign_pool"] and details["reserved_bytes"] > 0
|
| 175 |
+
return {"graph": graph, "static_input": static_input, "static_output": static_output,
|
| 176 |
+
"stream": current, "side": side, "pool": pool,
|
| 177 |
+
"pool_at_capture": details,
|
| 178 |
+
"static_input_storage_bytes": static_input.untyped_storage().nbytes(),
|
| 179 |
+
"capture_mutation_restored": True}
|
| 180 |
+
|
| 181 |
+
def replay_clone(self, entry: dict[str, Any], value: Any) -> Any:
|
| 182 |
+
if self.torch.cuda.current_stream(value.device).cuda_stream != entry["stream"].cuda_stream:
|
| 183 |
+
raise RuntimeError("Replay stream changed")
|
| 184 |
+
entry["static_input"].copy_(value)
|
| 185 |
+
entry["graph"].replay()
|
| 186 |
+
return entry["static_output"].clone()
|
| 187 |
+
|
| 188 |
+
def release(self, entry: dict[str, Any]) -> dict[str, Any]:
|
| 189 |
+
torch = self.torch
|
| 190 |
+
refs = {key: weakref.ref(entry[key]) for key in ("static_input", "static_output", "graph")}
|
| 191 |
+
events = []
|
| 192 |
+
|
| 193 |
+
def snapshot(stage: str) -> None:
|
| 194 |
+
item = describe_pool(torch.cuda.memory_snapshot(
|
| 195 |
+
mempool_id=entry["pool"], include_traces=False), entry["pool"])
|
| 196 |
+
item.update(stage=stage, weak_alive={key: ref() is not None for key, ref in refs.items()})
|
| 197 |
+
events.append(item)
|
| 198 |
+
|
| 199 |
+
entry["stream"].synchronize()
|
| 200 |
+
entry["side"].synchronize()
|
| 201 |
+
snapshot("before_drop")
|
| 202 |
+
entry["graph"].reset()
|
| 203 |
+
entry["static_input"] = entry["static_output"] = entry["graph"] = None
|
| 204 |
+
snapshot("after_drop")
|
| 205 |
+
entry["stream"].synchronize()
|
| 206 |
+
entry["side"].synchronize()
|
| 207 |
+
with torch.cuda.stream(entry["stream"]):
|
| 208 |
+
probe = torch.empty((1,), device=entry["stream"].device, dtype=torch.uint8)
|
| 209 |
+
del probe
|
| 210 |
+
entry["stream"].synchronize()
|
| 211 |
+
snapshot("after_fence_allocator_poll")
|
| 212 |
+
final = events[-1]
|
| 213 |
+
checks = {"no_graph_refs": not any(final["weak_alive"].values()),
|
| 214 |
+
"correct_pool": not final["foreign_pool"],
|
| 215 |
+
"pool_active_zero": final["active_bytes"] == 0,
|
| 216 |
+
"pool_allocated_zero": final["allocated_bytes"] == 0,
|
| 217 |
+
"no_active_blocks": final["active_allocated_blocks"] == 0,
|
| 218 |
+
"no_pending_blocks": final["pending_blocks"] == 0,
|
| 219 |
+
"known_block_states": not final["unknown_block_states"]}
|
| 220 |
+
return {"passed": all(checks.values()), "checks": checks, "events": events}
|
clairic_runtime/decoder_graph.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generic startup GDN decoder graphs with owned native state prefixes."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from contextlib import contextmanager
|
| 5 |
+
import os
|
| 6 |
+
import threading
|
| 7 |
+
import time
|
| 8 |
+
from types import MethodType
|
| 9 |
+
from typing import Any, Iterator
|
| 10 |
+
import weakref
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
from .decoder_capture import DecoderGraph, TorchBackend, callable_key
|
| 15 |
+
from .gdn_pool import NativeStatePool
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class _SelectedCache:
|
| 19 |
+
"""Expose one real native layer to the proved single-layer capture adapter."""
|
| 20 |
+
|
| 21 |
+
def __init__(self, cache: Any, index: int) -> None:
|
| 22 |
+
self.cache, self.index = cache, index
|
| 23 |
+
self.layers = [cache.layers[index]]
|
| 24 |
+
|
| 25 |
+
def has_previous_state(self, index: int) -> bool:
|
| 26 |
+
assert index == 0
|
| 27 |
+
return self.cache.has_previous_state(self.index)
|
| 28 |
+
|
| 29 |
+
def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor:
|
| 30 |
+
assert index == 0
|
| 31 |
+
return self.cache.update_recurrent_state(state, self.index)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class DecoderGraphs:
|
| 35 |
+
"""One engine, CUDA stream and worker; native prefill and dense attention stay eager."""
|
| 36 |
+
|
| 37 |
+
def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None:
|
| 38 |
+
self.backbone = backbone
|
| 39 |
+
self.thread = threading.get_ident()
|
| 40 |
+
self.device = backbone.device
|
| 41 |
+
self.stream = torch.cuda.current_stream(self.device)
|
| 42 |
+
self.max_batch, self.buckets = max_batch, buckets
|
| 43 |
+
self.pool = NativeStatePool(backbone, max_batch)
|
| 44 |
+
self.graphs: dict[tuple[int, int], DecoderGraph] = {}
|
| 45 |
+
self.wrappers: list[tuple[Any, Any, Any]] = []
|
| 46 |
+
self.active: tuple[Any, int] | None = None
|
| 47 |
+
self.closed = False
|
| 48 |
+
self.preparing = True
|
| 49 |
+
self.stats = {"replay": 0, "eager": 0, "retired": 0}
|
| 50 |
+
self.cleanup: list[dict[str, Any]] = []
|
| 51 |
+
self.preparation: list[dict[str, Any]] = []
|
| 52 |
+
self.observe_requests = False
|
| 53 |
+
self.last_request: dict[str, Any] | None = None
|
| 54 |
+
self.active_methods: dict[int, Any] = {}
|
| 55 |
+
|
| 56 |
+
def _owner(self) -> bool:
|
| 57 |
+
return (not self.closed and threading.get_ident() == self.thread
|
| 58 |
+
and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream)
|
| 59 |
+
|
| 60 |
+
@staticmethod
|
| 61 |
+
def _runtime() -> tuple[Any, ...]:
|
| 62 |
+
import deep_gemm as dg
|
| 63 |
+
|
| 64 |
+
return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(),
|
| 65 |
+
tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_"))))
|
| 66 |
+
|
| 67 |
+
def prepare(self) -> None:
|
| 68 |
+
if not self._owner():
|
| 69 |
+
raise RuntimeError("Graph preparation must use the engine CUDA worker")
|
| 70 |
+
backend = TorchBackend(torch, self._runtime)
|
| 71 |
+
try:
|
| 72 |
+
for index in self.pool.states:
|
| 73 |
+
layer = self.pool.language.layers[index]
|
| 74 |
+
eager = layer.forward
|
| 75 |
+
weak_manager = weakref.ref(self)
|
| 76 |
+
|
| 77 |
+
def forward(this: Any, *args: Any, _index: int = index,
|
| 78 |
+
_eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any:
|
| 79 |
+
manager = _manager()
|
| 80 |
+
if manager is None:
|
| 81 |
+
return _eager(*args, **kwargs)
|
| 82 |
+
return manager._forward(_index, _eager, args, kwargs)
|
| 83 |
+
|
| 84 |
+
installed = MethodType(forward, layer)
|
| 85 |
+
self.wrappers.append((layer, eager, installed))
|
| 86 |
+
layer.forward = installed
|
| 87 |
+
dtype = torch.bfloat16 if index == 0 else torch.float32
|
| 88 |
+
for bucket in self.buckets:
|
| 89 |
+
began = time.perf_counter_ns()
|
| 90 |
+
dummy = self.pool.new_cache(bucket, previous=True)
|
| 91 |
+
selected = _SelectedCache(dummy, index)
|
| 92 |
+
|
| 93 |
+
def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any:
|
| 94 |
+
return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None)
|
| 95 |
+
|
| 96 |
+
zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(),
|
| 97 |
+
dtype=dtype, device=self.device)
|
| 98 |
+
self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero,
|
| 99 |
+
operation, exclusive_process=True)
|
| 100 |
+
entry = self.graphs[index, bucket].entry
|
| 101 |
+
self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype),
|
| 102 |
+
"capture_ms": (time.perf_counter_ns() - began) / 1e6,
|
| 103 |
+
"generic_zero_input": True,
|
| 104 |
+
"capture_mutation_restored": entry["capture_mutation_restored"],
|
| 105 |
+
"pool": list(entry["pool"]),
|
| 106 |
+
"pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"],
|
| 107 |
+
"static_input_storage_bytes": entry["static_input_storage_bytes"]})
|
| 108 |
+
self.pool.zero()
|
| 109 |
+
self.stream.synchronize()
|
| 110 |
+
self.preparing = False
|
| 111 |
+
except BaseException:
|
| 112 |
+
self.close()
|
| 113 |
+
raise
|
| 114 |
+
|
| 115 |
+
@contextmanager
|
| 116 |
+
def request(self, batch: int) -> Iterator[Any]:
|
| 117 |
+
if (not self._owner() or self.preparing or self.active is not None
|
| 118 |
+
or not 1 <= batch <= self.max_batch or torch.is_grad_enabled()
|
| 119 |
+
or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()):
|
| 120 |
+
yield None
|
| 121 |
+
return
|
| 122 |
+
cache = self.pool.new_cache(batch)
|
| 123 |
+
if not self.pool.owns(cache, batch):
|
| 124 |
+
raise RuntimeError("Fresh cache must use owned native state prefixes")
|
| 125 |
+
self.active = (cache, batch)
|
| 126 |
+
self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states}
|
| 127 |
+
completed = False
|
| 128 |
+
try:
|
| 129 |
+
yield cache
|
| 130 |
+
completed = True
|
| 131 |
+
finally:
|
| 132 |
+
try:
|
| 133 |
+
if self.observe_requests:
|
| 134 |
+
self.last_request = {"batch": batch, "completed": completed,
|
| 135 |
+
"cache_type": type(cache).__name__,
|
| 136 |
+
"native_prefixes_owned": self.pool.owns(cache, batch),
|
| 137 |
+
"layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape),
|
| 138 |
+
"conv_stride": list(cache.layers[index].conv_states.stride()),
|
| 139 |
+
"conv_dtype": str(cache.layers[index].conv_states.dtype),
|
| 140 |
+
"recurrent_shape": list(cache.layers[index].recurrent_states.shape),
|
| 141 |
+
"recurrent_stride": list(cache.layers[index].recurrent_states.stride()),
|
| 142 |
+
"recurrent_dtype": str(cache.layers[index].recurrent_states.dtype),
|
| 143 |
+
"has_previous_state": bool(cache.layers[index].has_previous_state)}
|
| 144 |
+
for index in self.pool.states]}
|
| 145 |
+
finally:
|
| 146 |
+
self.active = None
|
| 147 |
+
self.active_methods.clear()
|
| 148 |
+
|
| 149 |
+
@staticmethod
|
| 150 |
+
def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]:
|
| 151 |
+
layer = cache.layers[index]
|
| 152 |
+
return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state),
|
| 153 |
+
callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state),
|
| 154 |
+
layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size)
|
| 155 |
+
|
| 156 |
+
def diagnostics(self) -> dict[str, Any]:
|
| 157 |
+
"""Primitive-only snapshot for separate untimed diagnostics and shutdown."""
|
| 158 |
+
return {"closed": self.closed, "preparing": self.preparing,
|
| 159 |
+
"active": self.active is not None, "owner_thread": self.thread,
|
| 160 |
+
"stream": self.stream.cuda_stream, "max_batch": self.max_batch,
|
| 161 |
+
"buckets": list(self.buckets), "graphs": len(self.graphs),
|
| 162 |
+
"arena_storage_bytes": self.pool.storage_bytes,
|
| 163 |
+
"stats": dict(self.stats), "preparation": list(self.preparation),
|
| 164 |
+
"observe_requests": self.observe_requests,
|
| 165 |
+
"last_request": self.last_request, "cleanup": list(self.cleanup)}
|
| 166 |
+
|
| 167 |
+
def _retire(self, index: int) -> None:
|
| 168 |
+
for key in tuple(self.graphs):
|
| 169 |
+
if key[0] == index:
|
| 170 |
+
graph = self.graphs[key]
|
| 171 |
+
details = graph.close()
|
| 172 |
+
del self.graphs[key]
|
| 173 |
+
self.cleanup.append({"layer": key[0], "bucket": key[1], **details})
|
| 174 |
+
if not details["passed"]:
|
| 175 |
+
raise RuntimeError("Decoder graph retirement failed strict cleanup")
|
| 176 |
+
self.stats["retired"] += 1
|
| 177 |
+
|
| 178 |
+
def _owns_layer(self, cache: Any, batch: int, index: int) -> bool:
|
| 179 |
+
layer = cache.layers[index]
|
| 180 |
+
for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True):
|
| 181 |
+
if (actual is None or actual.data_ptr() != owned.data_ptr()
|
| 182 |
+
or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
|
| 183 |
+
or actual.dtype != owned.dtype or actual.device != owned.device
|
| 184 |
+
or actual.storage_offset() != owned.storage_offset()
|
| 185 |
+
or tuple(actual.shape) != (batch, *owned.shape[1:])
|
| 186 |
+
or tuple(actual.stride()) != tuple(owned.stride())):
|
| 187 |
+
return False
|
| 188 |
+
return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized
|
| 189 |
+
|
| 190 |
+
def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any:
|
| 191 |
+
value = args[0] if args else kwargs.get("hidden_states")
|
| 192 |
+
cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None)
|
| 193 |
+
mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None)
|
| 194 |
+
batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0
|
| 195 |
+
expected_dtype = torch.bfloat16 if index == 0 else torch.float32
|
| 196 |
+
admitted = (not self.preparing and self._owner() and self.active is not None
|
| 197 |
+
and cache is self.active[0] and batch == self.active[1]
|
| 198 |
+
and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype
|
| 199 |
+
and value.device == self.device and mask is None and not torch.is_grad_enabled()
|
| 200 |
+
and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
|
| 201 |
+
and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids",
|
| 202 |
+
"past_key_values", "use_cache", "cache_position"}
|
| 203 |
+
and self._cache_methods(cache, index) == self.active_methods.get(index)
|
| 204 |
+
and self._owns_layer(cache, batch, index) and cache.has_previous_state(index))
|
| 205 |
+
if not admitted:
|
| 206 |
+
self.stats["eager"] += 1
|
| 207 |
+
return eager(*args, **kwargs)
|
| 208 |
+
bucket = next((bound for bound in self.buckets if batch <= bound), None)
|
| 209 |
+
graph = self.graphs.get((index, bucket))
|
| 210 |
+
if graph is None:
|
| 211 |
+
self.stats["eager"] += 1
|
| 212 |
+
return eager(*args, **kwargs)
|
| 213 |
+
entry = graph.entry
|
| 214 |
+
try:
|
| 215 |
+
graph.check(entry["static_input"])
|
| 216 |
+
except RuntimeError:
|
| 217 |
+
self._retire(index)
|
| 218 |
+
self.stats["eager"] += 1
|
| 219 |
+
return eager(*args, **kwargs)
|
| 220 |
+
entry["static_input"][:batch].copy_(value)
|
| 221 |
+
if batch < bucket:
|
| 222 |
+
entry["static_input"][batch:].zero_()
|
| 223 |
+
entry["graph"].replay()
|
| 224 |
+
self.stats["replay"] += 1
|
| 225 |
+
return entry["static_output"][:batch].clone()
|
| 226 |
+
|
| 227 |
+
def close(self) -> None:
|
| 228 |
+
if self.closed:
|
| 229 |
+
return
|
| 230 |
+
if not self._owner():
|
| 231 |
+
raise RuntimeError("Graph cleanup must use the engine CUDA worker")
|
| 232 |
+
self.active = None
|
| 233 |
+
self.active_methods.clear()
|
| 234 |
+
error = None
|
| 235 |
+
try:
|
| 236 |
+
for index in tuple(self.pool.states):
|
| 237 |
+
try:
|
| 238 |
+
self._retire(index)
|
| 239 |
+
except BaseException as exc:
|
| 240 |
+
if error is None:
|
| 241 |
+
error = exc
|
| 242 |
+
finally:
|
| 243 |
+
for layer, original, installed in reversed(self.wrappers):
|
| 244 |
+
if layer.forward is installed:
|
| 245 |
+
layer.forward = original
|
| 246 |
+
self.wrappers.clear()
|
| 247 |
+
self.pool.close()
|
| 248 |
+
self.backbone = None
|
| 249 |
+
self.closed = True
|
| 250 |
+
if error is not None:
|
| 251 |
+
raise error
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def _admissible(backbone: Any, max_batch: int) -> bool:
|
| 255 |
+
"""Reject unsupported contexts before allocating arenas or installing wrappers."""
|
| 256 |
+
config = backbone.config.get_text_config()
|
| 257 |
+
limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0
|
| 258 |
+
if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16
|
| 259 |
+
or not 1 <= max_batch <= limit):
|
| 260 |
+
return False
|
| 261 |
+
registry = torch.nn.modules.module
|
| 262 |
+
if registry._global_forward_hooks or registry._global_forward_pre_hooks:
|
| 263 |
+
return False
|
| 264 |
+
for module in backbone.modules():
|
| 265 |
+
if module._forward_hooks or module._forward_pre_hooks:
|
| 266 |
+
return False
|
| 267 |
+
for layer in backbone.model.language_model.layers:
|
| 268 |
+
if layer.layer_type != "linear_attention":
|
| 269 |
+
continue
|
| 270 |
+
for value in layer.parameters():
|
| 271 |
+
try:
|
| 272 |
+
version = value._version
|
| 273 |
+
except RuntimeError:
|
| 274 |
+
return False
|
| 275 |
+
if version is None:
|
| 276 |
+
return False
|
| 277 |
+
return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None:
|
| 281 |
+
"""Run on the existing worker after math installation; unsupported engines stay eager."""
|
| 282 |
+
if not _admissible(backbone, max_batch):
|
| 283 |
+
return None
|
| 284 |
+
largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound)
|
| 285 |
+
buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest)
|
| 286 |
+
with torch.no_grad():
|
| 287 |
+
manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets)
|
| 288 |
+
manager.prepare()
|
| 289 |
+
return manager
|
clairic_runtime/gdn/LICENSES
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Flash Linear Attention — MIT
|
| 2 |
+
============================
|
| 3 |
+
|
| 4 |
+
MIT License
|
| 5 |
+
|
| 6 |
+
Copyright (c) 2023-2025 Songlin Yang
|
| 7 |
+
|
| 8 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 9 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 10 |
+
in the Software without restriction, including without limitation the rights
|
| 11 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 12 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 13 |
+
furnished to do so, subject to the following conditions:
|
| 14 |
+
|
| 15 |
+
The above copyright notice and this permission notice shall be included in all
|
| 16 |
+
copies or substantial portions of the Software.
|
| 17 |
+
|
| 18 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 19 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 20 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 21 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 22 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 23 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 24 |
+
SOFTWARE.
|
| 25 |
+
|
| 26 |
+
SGLang — Apache-2.0
|
| 27 |
+
===================
|
| 28 |
+
|
| 29 |
+
Apache License
|
| 30 |
+
Version 2.0, January 2004
|
| 31 |
+
http://www.apache.org/licenses/
|
| 32 |
+
|
| 33 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 34 |
+
|
| 35 |
+
1. Definitions.
|
| 36 |
+
|
| 37 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 38 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 39 |
+
|
| 40 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 41 |
+
the copyright owner that is granting the License.
|
| 42 |
+
|
| 43 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 44 |
+
other entities that control, are controlled by, or are under common
|
| 45 |
+
control with that entity. For the purposes of this definition,
|
| 46 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 47 |
+
direction or management of such entity, whether by contract or
|
| 48 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 49 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 50 |
+
|
| 51 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 52 |
+
exercising permissions granted by this License.
|
| 53 |
+
|
| 54 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 55 |
+
including but not limited to software source code, documentation
|
| 56 |
+
source, and configuration files.
|
| 57 |
+
|
| 58 |
+
"Object" form shall mean any form resulting from mechanical
|
| 59 |
+
transformation or translation of a Source form, including but
|
| 60 |
+
not limited to compiled object code, generated documentation,
|
| 61 |
+
and conversions to other media types.
|
| 62 |
+
|
| 63 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 64 |
+
Object form, made available under the License, as indicated by a
|
| 65 |
+
copyright notice that is included in or attached to the work
|
| 66 |
+
(an example is provided in the Appendix below).
|
| 67 |
+
|
| 68 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 69 |
+
form, that is based on (or derived from) the Work and for which the
|
| 70 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 71 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 72 |
+
of this License, Derivative Works shall not include works that remain
|
| 73 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 74 |
+
the Work and Derivative Works thereof.
|
| 75 |
+
|
| 76 |
+
"Contribution" shall mean any work of authorship, including
|
| 77 |
+
the original version of the Work and any modifications or additions
|
| 78 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 79 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 80 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 81 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 82 |
+
means any form of electronic, verbal, or written communication sent
|
| 83 |
+
to the Licensor or its representatives, including but not limited to
|
| 84 |
+
communication on electronic mailing lists, source code control systems,
|
| 85 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 86 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 87 |
+
excluding communication that is conspicuously marked or otherwise
|
| 88 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 89 |
+
|
| 90 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 91 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 92 |
+
subsequently incorporated within the Work.
|
| 93 |
+
|
| 94 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 95 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 96 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 97 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 98 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 99 |
+
Work and such Derivative Works in Source or Object form.
|
| 100 |
+
|
| 101 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 102 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 103 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 104 |
+
(except as stated in this section) patent license to make, have made,
|
| 105 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 106 |
+
where such license applies only to those patent claims licensable
|
| 107 |
+
by such Contributor that are necessarily infringed by their
|
| 108 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 109 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 110 |
+
institute patent litigation against any entity (including a
|
| 111 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 112 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 113 |
+
or contributory patent infringement, then any patent licenses
|
| 114 |
+
granted to You under this License for that Work shall terminate
|
| 115 |
+
as of the date such litigation is filed.
|
| 116 |
+
|
| 117 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 118 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 119 |
+
modifications, and in Source or Object form, provided that You
|
| 120 |
+
meet the following conditions:
|
| 121 |
+
|
| 122 |
+
(a) You must give any other recipients of the Work or
|
| 123 |
+
Derivative Works a copy of this License; and
|
| 124 |
+
|
| 125 |
+
(b) You must cause any modified files to carry prominent notices
|
| 126 |
+
stating that You changed the files; and
|
| 127 |
+
|
| 128 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 129 |
+
that You distribute, all copyright, patent, trademark, and
|
| 130 |
+
attribution notices from the Source form of the Work,
|
| 131 |
+
excluding those notices that do not pertain to any part of
|
| 132 |
+
the Derivative Works; and
|
| 133 |
+
|
| 134 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 135 |
+
distribution, then any Derivative Works that You distribute must
|
| 136 |
+
include a readable copy of the attribution notices contained
|
| 137 |
+
within such NOTICE file, excluding those notices that do not
|
| 138 |
+
pertain to any part of the Derivative Works, in at least one
|
| 139 |
+
of the following places: within a NOTICE text file distributed
|
| 140 |
+
as part of the Derivative Works; within the Source form or
|
| 141 |
+
documentation, if provided along with the Derivative Works; or,
|
| 142 |
+
within a display generated by the Derivative Works, if and
|
| 143 |
+
wherever such third-party notices normally appear. The contents
|
| 144 |
+
of the NOTICE file are for informational purposes only and
|
| 145 |
+
do not modify the License. You may add Your own attribution
|
| 146 |
+
notices within Derivative Works that You distribute, alongside
|
| 147 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 148 |
+
that such additional attribution notices cannot be construed
|
| 149 |
+
as modifying the License.
|
| 150 |
+
|
| 151 |
+
You may add Your own copyright statement to Your modifications and
|
| 152 |
+
may provide additional or different license terms and conditions
|
| 153 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 154 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 155 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 156 |
+
the conditions stated in this License.
|
| 157 |
+
|
| 158 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 159 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 160 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 161 |
+
this License, without any additional terms or conditions.
|
| 162 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 163 |
+
the terms of any separate license agreement you may have executed
|
| 164 |
+
with Licensor regarding such Contributions.
|
| 165 |
+
|
| 166 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 167 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 168 |
+
except as required for reasonable and customary use in describing the
|
| 169 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 170 |
+
|
| 171 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 172 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 173 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 174 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 175 |
+
implied, including, without limitation, any warranties or conditions
|
| 176 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 177 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 178 |
+
appropriateness of using or redistributing the Work and assume any
|
| 179 |
+
risks associated with Your exercise of permissions under this License.
|
| 180 |
+
|
| 181 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 182 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 183 |
+
unless required by applicable law (such as deliberate and grossly
|
| 184 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 185 |
+
liable to You for damages, including any direct, indirect, special,
|
| 186 |
+
incidental, or consequential damages of any character arising as a
|
| 187 |
+
result of this License or out of the use or inability to use the
|
| 188 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 189 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 190 |
+
other commercial damages or losses), even if such Contributor
|
| 191 |
+
has been advised of the possibility of such damages.
|
| 192 |
+
|
| 193 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 194 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 195 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 196 |
+
or other liability obligations and/or rights consistent with this
|
| 197 |
+
License. However, in accepting such obligations, You may act only
|
| 198 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 199 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 200 |
+
defend, and hold each Contributor harmless for any liability
|
| 201 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 202 |
+
of your accepting any such warranty or additional liability.
|
| 203 |
+
|
| 204 |
+
END OF TERMS AND CONDITIONS
|
| 205 |
+
|
| 206 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 207 |
+
|
| 208 |
+
To apply the Apache License to your work, attach the following
|
| 209 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 210 |
+
replaced with your own identifying information. (Don't include
|
| 211 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 212 |
+
comment syntax for the file format. We also recommend that a
|
| 213 |
+
file or class name and description of purpose be included on the
|
| 214 |
+
same "printed page" as the copyright notice for easier
|
| 215 |
+
identification within third-party archives.
|
| 216 |
+
|
| 217 |
+
Copyright 2023-2024 SGLang Team
|
| 218 |
+
|
| 219 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 220 |
+
you may not use this file except in compliance with the License.
|
| 221 |
+
You may obtain a copy of the License at
|
| 222 |
+
|
| 223 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 224 |
+
|
| 225 |
+
Unless required by applicable law or agreed to in writing, software
|
| 226 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 227 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 228 |
+
See the License for the specific language governing permissions and
|
| 229 |
+
limitations under the License.
|
clairic_runtime/gdn/NOTICE
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Standalone GDN inference helpers
|
| 2 |
+
|
| 3 |
+
These files contain portions of Flash Linear Attention (FLA),
|
| 4 |
+
https://github.com/fla-org/flash-linear-attention, under the MIT License.
|
| 5 |
+
Copyright (c) 2023-2025 Songlin Yang.
|
| 6 |
+
The retained source headers also credit Songlin Yang and Yu Zhang.
|
| 7 |
+
|
| 8 |
+
They include adaptations from SGLang,
|
| 9 |
+
https://github.com/sgl-project/sglang, under the Apache License, Version 2.0.
|
| 10 |
+
Applicable original source headers are retained.
|
| 11 |
+
|
| 12 |
+
The module import layout and standalone helper plumbing were adapted.
|
| 13 |
+
The selected inference kernel bodies and launch configurations are retained.
|
| 14 |
+
Full license texts are provided in LICENSES.
|
clairic_runtime/gdn/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Gated delta network inference operations."""
|
clairic_runtime/gdn/adapter.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Pack native inputs and preserve native cache axes around selected kernels."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
from .. import recurrent_metadata
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
|
| 13 |
+
"""Return valid-token indices and one boundary per native row."""
|
| 14 |
+
batch, length = mixed.shape[:2]
|
| 15 |
+
if mask is None:
|
| 16 |
+
indices = torch.arange(batch * length, device=mixed.device)
|
| 17 |
+
boundaries = torch.arange(batch + 1, device=mixed.device, dtype=torch.int32) * length
|
| 18 |
+
return indices, boundaries
|
| 19 |
+
if mask.ndim != 2 or mask.shape[0] != batch or mask.shape[1] < length:
|
| 20 |
+
raise ValueError("Expected a two-dimensional mask covering current tokens")
|
| 21 |
+
valid = mask[:, -length:].to(device=mixed.device, dtype=torch.bool)
|
| 22 |
+
lengths = valid.sum(dim=1, dtype=torch.int32)
|
| 23 |
+
if bool((lengths == 0).any()):
|
| 24 |
+
raise ValueError("Each recurrent sequence must contain a current token")
|
| 25 |
+
boundaries = torch.cat((lengths.new_zeros(1), lengths.cumsum(0, dtype=torch.int32)))
|
| 26 |
+
return valid.reshape(-1).nonzero().flatten(), boundaries
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
|
| 30 |
+
"""Reuse exact packing metadata within an owned inference forward."""
|
| 31 |
+
return recurrent_metadata.packed_indices(mixed, mask, _packed_indices)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def source_state(native: torch.Tensor | None, batch: int, heads: int, key_dim: int,
|
| 35 |
+
value_dim: int, reference: torch.Tensor) -> torch.Tensor:
|
| 36 |
+
"""Map native K,V cache dimensions to the selected kernel's V,K layout."""
|
| 37 |
+
if native is None:
|
| 38 |
+
return torch.zeros(batch, heads, value_dim, key_dim,
|
| 39 |
+
device=reference.device, dtype=torch.float32)
|
| 40 |
+
if tuple(native.shape) != (batch, heads, key_dim, value_dim):
|
| 41 |
+
raise ValueError("Unexpected native recurrent cache shape")
|
| 42 |
+
if native.dtype != torch.float32:
|
| 43 |
+
raise ValueError("Recurrent states must retain float32 precision")
|
| 44 |
+
return native.transpose(-1, -2).contiguous()
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def native_state(source: torch.Tensor) -> torch.Tensor:
|
| 48 |
+
"""Expose K,V cache axes without narrowing or copying source state."""
|
| 49 |
+
return source.transpose(-1, -2)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def store_state(cache: Any, layer_idx: int, state: torch.Tensor) -> torch.Tensor:
|
| 53 |
+
"""Initialize FP32 recurrent storage while retaining native cache lifecycle."""
|
| 54 |
+
if state.dtype != torch.float32:
|
| 55 |
+
raise ValueError("Expected float32 recurrent state")
|
| 56 |
+
layer = cache.layers[layer_idx]
|
| 57 |
+
if not layer.is_recurrent_states_initialized:
|
| 58 |
+
layer.recurrent_states = torch.empty_like(state)
|
| 59 |
+
if not torch.compiler.is_compiling():
|
| 60 |
+
torch._dynamo.mark_static_address(layer.recurrent_states)
|
| 61 |
+
layer.is_recurrent_states_initialized = True
|
| 62 |
+
elif layer.recurrent_states.dtype != torch.float32:
|
| 63 |
+
raise ValueError("Cannot resume a narrowed recurrent cache")
|
| 64 |
+
return cache.update_recurrent_state(state, layer_idx)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def prefill(mixed: torch.Tensor, a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor,
|
| 68 |
+
dt_bias: torch.Tensor, key_heads: int, value_heads: int, key_dim: int,
|
| 69 |
+
value_dim: int, initial_state: torch.Tensor | None,
|
| 70 |
+
attention_mask: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
|
| 71 |
+
"""Run packed prefill and scatter valid outputs to native padded shape."""
|
| 72 |
+
from .gating import fused_gdn_gating
|
| 73 |
+
from .index import prepare_chunk_indices
|
| 74 |
+
from .l2norm import l2norm_fwd
|
| 75 |
+
from .prefill import chunk_gated_delta_rule_fwd
|
| 76 |
+
|
| 77 |
+
batch, length, _ = mixed.shape
|
| 78 |
+
indices, boundaries = packed_indices(mixed, attention_mask)
|
| 79 |
+
packed = mixed.flatten(0, 1).index_select(0, indices)
|
| 80 |
+
pa = a.flatten(0, 1).index_select(0, indices)
|
| 81 |
+
pb = b.flatten(0, 1).index_select(0, indices)
|
| 82 |
+
query, key, value = torch.split(packed,
|
| 83 |
+
(key_heads * key_dim, key_heads * key_dim, value_heads * value_dim), dim=-1)
|
| 84 |
+
query = l2norm_fwd(query.reshape(1, -1, key_heads, key_dim).contiguous())
|
| 85 |
+
key = l2norm_fwd(key.reshape(1, -1, key_heads, key_dim).contiguous())
|
| 86 |
+
value = value.reshape(1, -1, value_heads, value_dim).contiguous()
|
| 87 |
+
gate, beta = fused_gdn_gating(A_log, pa, pb, dt_bias)
|
| 88 |
+
state = source_state(initial_state, batch, value_heads, key_dim, value_dim, mixed)
|
| 89 |
+
slots = recurrent_metadata.sequence_slots(mixed)
|
| 90 |
+
_, output, *_ = chunk_gated_delta_rule_fwd(query, key, value, gate, beta,
|
| 91 |
+
key_dim ** -0.5, state, slots, boundaries, prepare_chunk_indices(boundaries, 64))
|
| 92 |
+
padded = mixed.new_zeros(batch * length, value_heads, value_dim)
|
| 93 |
+
padded.index_copy_(0, indices, output.squeeze(0))
|
| 94 |
+
return padded.reshape(batch, length, value_heads, value_dim), native_state(state)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def decode(mixed: torch.Tensor, a: torch.Tensor, b: torch.Tensor, A_log: torch.Tensor,
|
| 98 |
+
dt_bias: torch.Tensor, initial_state: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 99 |
+
"""Use the selected packed single-token recurrent operation."""
|
| 100 |
+
from .recurrent import fused_recurrent_gated_delta_rule_packed_decode
|
| 101 |
+
|
| 102 |
+
batch, length, _ = mixed.shape
|
| 103 |
+
if length != 1:
|
| 104 |
+
raise ValueError("Packed recurrent decode requires one token per row")
|
| 105 |
+
_, heads, key_dim, value_dim = initial_state.shape
|
| 106 |
+
state = source_state(initial_state, batch, heads, key_dim, value_dim, mixed)
|
| 107 |
+
output = mixed.new_empty(batch, 1, heads, value_dim)
|
| 108 |
+
slots = recurrent_metadata.sequence_slots(mixed)
|
| 109 |
+
fused_recurrent_gated_delta_rule_packed_decode(mixed[:, 0].contiguous(),
|
| 110 |
+
a[:, 0].contiguous(), b[:, 0].contiguous(), A_log, dt_bias, key_dim ** -0.5,
|
| 111 |
+
state, output, slots, use_qk_l2norm_in_kernel=True)
|
| 112 |
+
return output, native_state(state)
|
clairic_runtime/gdn/chunk_delta_h.py
ADDED
|
@@ -0,0 +1,386 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_delta_h.py
|
| 3 |
+
# -*- coding: utf-8 -*-
|
| 4 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 5 |
+
|
| 6 |
+
from typing import Optional, Tuple
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import triton
|
| 10 |
+
import triton.language as tl
|
| 11 |
+
|
| 12 |
+
from .index import (
|
| 13 |
+
prepare_chunk_indices,
|
| 14 |
+
prepare_chunk_offsets,
|
| 15 |
+
)
|
| 16 |
+
from .op import exp, exp2, safe_exp
|
| 17 |
+
from .utils import (
|
| 18 |
+
autotune_cache_kwargs,
|
| 19 |
+
is_nvidia_hopper,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16]
|
| 23 |
+
CHUNK_SIZE = 64
|
| 24 |
+
GDN_CHUNK_H_BV = 32
|
| 25 |
+
GDN_CHUNK_H_NUM_WARPS = 4
|
| 26 |
+
GDN_CHUNK_H_NUM_STAGES = 2
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
@triton.autotune(
|
| 30 |
+
# Single hardcoded config. The kernel writes ht (final state) back into
|
| 31 |
+
# initial_state in-place; with multiple configs, triton's autotuning
|
| 32 |
+
# phase invokes the kernel many times for timing and corrupts the cache pool,
|
| 33 |
+
# producing silently wrong output on the first user request. Restoring via
|
| 34 |
+
# `restore_value=["initial_state"]` works for unit tests but OOMs on
|
| 35 |
+
# production-scale models (e.g. Kimi-Linear-48B at default mem_fraction)
|
| 36 |
+
# because cloning the cache pool for each configuration exceeds available memory.
|
| 37 |
+
# NT_BUCKET is kept in the autotune key for forward-compatibility (allows
|
| 38 |
+
# future per-bucket configs once the kernel is refactored to write final
|
| 39 |
+
# state to a separate output buffer). The env knobs keep this single-config
|
| 40 |
+
# property while allowing model/hardware-local validation of the selected
|
| 41 |
+
# tile without corrupting the state pool through multi-config autotune.
|
| 42 |
+
configs=[
|
| 43 |
+
triton.Config(
|
| 44 |
+
{"BV": GDN_CHUNK_H_BV},
|
| 45 |
+
num_warps=GDN_CHUNK_H_NUM_WARPS,
|
| 46 |
+
num_stages=GDN_CHUNK_H_NUM_STAGES,
|
| 47 |
+
)
|
| 48 |
+
],
|
| 49 |
+
key=["H", "K", "V", "BT", "USE_GK", "NT_BUCKET"],
|
| 50 |
+
**autotune_cache_kwargs,
|
| 51 |
+
)
|
| 52 |
+
@triton.jit(do_not_specialize=["T"])
|
| 53 |
+
def chunk_gated_delta_rule_fwd_kernel_h_blockdim64(
|
| 54 |
+
k: tl.tensor,
|
| 55 |
+
v: tl.tensor,
|
| 56 |
+
w: tl.tensor,
|
| 57 |
+
v_new: tl.tensor,
|
| 58 |
+
g: tl.tensor,
|
| 59 |
+
gk: tl.tensor,
|
| 60 |
+
h: tl.tensor,
|
| 61 |
+
initial_state: tl.tensor,
|
| 62 |
+
initial_state_indices: tl.tensor,
|
| 63 |
+
stride_init_state: tl.tensor,
|
| 64 |
+
cu_seqlens: tl.tensor,
|
| 65 |
+
chunk_offsets: tl.tensor,
|
| 66 |
+
T: tl.int32,
|
| 67 |
+
H: tl.constexpr,
|
| 68 |
+
Hg: tl.constexpr,
|
| 69 |
+
K: tl.constexpr,
|
| 70 |
+
V: tl.constexpr,
|
| 71 |
+
BT: tl.constexpr,
|
| 72 |
+
BV: tl.constexpr,
|
| 73 |
+
USE_G: tl.constexpr,
|
| 74 |
+
USE_GK: tl.constexpr,
|
| 75 |
+
USE_INITIAL_STATE: tl.constexpr,
|
| 76 |
+
INPLACE_UPDATE: tl.constexpr,
|
| 77 |
+
SAVE_NEW_VALUE: tl.constexpr,
|
| 78 |
+
IS_VARLEN: tl.constexpr,
|
| 79 |
+
NT_BUCKET: tl.constexpr,
|
| 80 |
+
USE_EXP2: tl.constexpr,
|
| 81 |
+
) -> None:
|
| 82 |
+
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
| 83 |
+
i_n, i_h = i_nh // H, i_nh % H
|
| 84 |
+
if IS_VARLEN:
|
| 85 |
+
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
|
| 86 |
+
cu_seqlens + i_n + 1
|
| 87 |
+
).to(tl.int32)
|
| 88 |
+
T = eos - bos
|
| 89 |
+
NT = tl.cdiv(T, BT)
|
| 90 |
+
boh = tl.load(chunk_offsets + i_n).to(tl.int32)
|
| 91 |
+
else:
|
| 92 |
+
bos, eos = i_n * T, i_n * T + T
|
| 93 |
+
NT = tl.cdiv(T, BT)
|
| 94 |
+
boh = i_n * NT
|
| 95 |
+
|
| 96 |
+
# [BV, BK]
|
| 97 |
+
b_h1 = tl.zeros([BV, 64], dtype=tl.float32)
|
| 98 |
+
if K > 64:
|
| 99 |
+
b_h2 = tl.zeros([BV, 64], dtype=tl.float32)
|
| 100 |
+
if K > 128:
|
| 101 |
+
b_h3 = tl.zeros([BV, 64], dtype=tl.float32)
|
| 102 |
+
if K > 192:
|
| 103 |
+
b_h4 = tl.zeros([BV, 64], dtype=tl.float32)
|
| 104 |
+
|
| 105 |
+
# calculate offset
|
| 106 |
+
h += ((boh * H + i_h) * V * K).to(tl.int64)
|
| 107 |
+
v += ((bos * H + i_h) * V).to(tl.int64)
|
| 108 |
+
k += ((bos * Hg + i_h // (H // Hg)) * K).to(tl.int64)
|
| 109 |
+
w += ((bos * H + i_h) * K).to(tl.int64)
|
| 110 |
+
if SAVE_NEW_VALUE:
|
| 111 |
+
v_new += ((bos * H + i_h) * V).to(tl.int64)
|
| 112 |
+
stride_v = H * V
|
| 113 |
+
stride_h = H * V * K
|
| 114 |
+
stride_k = Hg * K
|
| 115 |
+
stride_w = H * K
|
| 116 |
+
|
| 117 |
+
# Slot stride comes from the caller (initial_state.stride(0)): the state pool
|
| 118 |
+
# may be an envelope-strided view (page-major / unified memory), where the
|
| 119 |
+
# per-slot pitch spans ALL layers' state, not H*V*K. int64: envelope pitches
|
| 120 |
+
# overflow an int32 index product.
|
| 121 |
+
index = tl.load(initial_state_indices + i_n).to(tl.int64)
|
| 122 |
+
# Padded rows carry the -1 sentinel; the decode kernel guards on it
|
| 123 |
+
# (fused_recurrent.py), the chunked extend path did not.
|
| 124 |
+
valid_state = index >= 0
|
| 125 |
+
h0 = initial_state + index * stride_init_state
|
| 126 |
+
ht = initial_state + index * stride_init_state
|
| 127 |
+
if USE_INITIAL_STATE:
|
| 128 |
+
h0 = h0 + i_h * V * K
|
| 129 |
+
if INPLACE_UPDATE:
|
| 130 |
+
ht = ht + i_h * V * K
|
| 131 |
+
|
| 132 |
+
# load initial state
|
| 133 |
+
if USE_INITIAL_STATE and valid_state:
|
| 134 |
+
p_h0_1 = tl.make_block_ptr(h0, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
|
| 135 |
+
b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32)
|
| 136 |
+
if K > 64:
|
| 137 |
+
p_h0_2 = tl.make_block_ptr(
|
| 138 |
+
h0, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
|
| 139 |
+
)
|
| 140 |
+
b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32)
|
| 141 |
+
if K > 128:
|
| 142 |
+
p_h0_3 = tl.make_block_ptr(
|
| 143 |
+
h0, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
|
| 144 |
+
)
|
| 145 |
+
b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32)
|
| 146 |
+
if K > 192:
|
| 147 |
+
p_h0_4 = tl.make_block_ptr(
|
| 148 |
+
h0, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
|
| 149 |
+
)
|
| 150 |
+
b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32)
|
| 151 |
+
|
| 152 |
+
# main recurrence
|
| 153 |
+
for i_t in range(NT):
|
| 154 |
+
p_h1 = tl.make_block_ptr(
|
| 155 |
+
h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0)
|
| 156 |
+
)
|
| 157 |
+
tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1))
|
| 158 |
+
if K > 64:
|
| 159 |
+
p_h2 = tl.make_block_ptr(
|
| 160 |
+
h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
|
| 161 |
+
)
|
| 162 |
+
tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1))
|
| 163 |
+
if K > 128:
|
| 164 |
+
p_h3 = tl.make_block_ptr(
|
| 165 |
+
h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
|
| 166 |
+
)
|
| 167 |
+
tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1))
|
| 168 |
+
if K > 192:
|
| 169 |
+
p_h4 = tl.make_block_ptr(
|
| 170 |
+
h + i_t * stride_h, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
|
| 171 |
+
)
|
| 172 |
+
tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1))
|
| 173 |
+
|
| 174 |
+
p_w = tl.make_block_ptr(
|
| 175 |
+
w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0)
|
| 176 |
+
)
|
| 177 |
+
b_w = tl.load(p_w, boundary_check=(0, 1))
|
| 178 |
+
b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype))
|
| 179 |
+
if K > 64:
|
| 180 |
+
p_w = tl.make_block_ptr(
|
| 181 |
+
w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0)
|
| 182 |
+
)
|
| 183 |
+
b_w = tl.load(p_w, boundary_check=(0, 1))
|
| 184 |
+
b_v += tl.dot(b_w, tl.trans(b_h2).to(b_w.dtype))
|
| 185 |
+
if K > 128:
|
| 186 |
+
p_w = tl.make_block_ptr(
|
| 187 |
+
w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0)
|
| 188 |
+
)
|
| 189 |
+
b_w = tl.load(p_w, boundary_check=(0, 1))
|
| 190 |
+
b_v += tl.dot(b_w, tl.trans(b_h3).to(b_w.dtype))
|
| 191 |
+
if K > 192:
|
| 192 |
+
p_w = tl.make_block_ptr(
|
| 193 |
+
w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0)
|
| 194 |
+
)
|
| 195 |
+
b_w = tl.load(p_w, boundary_check=(0, 1))
|
| 196 |
+
b_v += tl.dot(b_w, tl.trans(b_h4).to(b_w.dtype))
|
| 197 |
+
p_v = tl.make_block_ptr(
|
| 198 |
+
v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
|
| 199 |
+
)
|
| 200 |
+
b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v
|
| 201 |
+
|
| 202 |
+
if SAVE_NEW_VALUE:
|
| 203 |
+
p_v = tl.make_block_ptr(
|
| 204 |
+
v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
|
| 205 |
+
)
|
| 206 |
+
tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1))
|
| 207 |
+
|
| 208 |
+
last_idx = min((i_t + 1) * BT, T) - 1
|
| 209 |
+
if USE_G:
|
| 210 |
+
b_g_last = tl.load(g + bos * H + last_idx * H + i_h)
|
| 211 |
+
p_g = tl.make_block_ptr(
|
| 212 |
+
g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
|
| 213 |
+
)
|
| 214 |
+
b_g = tl.load(p_g, boundary_check=(0,))
|
| 215 |
+
b_v = b_v * safe_exp(b_g_last - b_g)[:, None]
|
| 216 |
+
b_g_last = exp(b_g_last)
|
| 217 |
+
b_h1 = b_h1 * b_g_last
|
| 218 |
+
if K > 64:
|
| 219 |
+
b_h2 = b_h2 * b_g_last
|
| 220 |
+
if K > 128:
|
| 221 |
+
b_h3 = b_h3 * b_g_last
|
| 222 |
+
if K > 192:
|
| 223 |
+
b_h4 = b_h4 * b_g_last
|
| 224 |
+
|
| 225 |
+
if USE_GK:
|
| 226 |
+
o_k1 = tl.arange(0, 64)
|
| 227 |
+
b_gk_last1 = tl.load(
|
| 228 |
+
gk + (bos + last_idx) * H * K + i_h * K + o_k1,
|
| 229 |
+
mask=(o_k1 < K),
|
| 230 |
+
other=0.0,
|
| 231 |
+
)
|
| 232 |
+
if USE_EXP2:
|
| 233 |
+
b_h1 *= exp2(b_gk_last1)[None, :]
|
| 234 |
+
else:
|
| 235 |
+
b_h1 *= exp(b_gk_last1)[None, :]
|
| 236 |
+
if K > 64:
|
| 237 |
+
o_k2 = 64 + o_k1
|
| 238 |
+
b_gk_last2 = tl.load(
|
| 239 |
+
gk + (bos + last_idx) * H * K + i_h * K + o_k2,
|
| 240 |
+
mask=(o_k2 < K),
|
| 241 |
+
other=0.0,
|
| 242 |
+
)
|
| 243 |
+
if USE_EXP2:
|
| 244 |
+
b_h2 *= exp2(b_gk_last2)[None, :]
|
| 245 |
+
else:
|
| 246 |
+
b_h2 *= exp(b_gk_last2)[None, :]
|
| 247 |
+
if K > 128:
|
| 248 |
+
o_k3 = 128 + o_k1
|
| 249 |
+
b_gk_last3 = tl.load(
|
| 250 |
+
gk + (bos + last_idx) * H * K + i_h * K + o_k3,
|
| 251 |
+
mask=(o_k3 < K),
|
| 252 |
+
other=0.0,
|
| 253 |
+
)
|
| 254 |
+
if USE_EXP2:
|
| 255 |
+
b_h3 *= exp2(b_gk_last3)[None, :]
|
| 256 |
+
else:
|
| 257 |
+
b_h3 *= exp(b_gk_last3)[None, :]
|
| 258 |
+
if K > 192:
|
| 259 |
+
o_k4 = 192 + o_k1
|
| 260 |
+
b_gk_last4 = tl.load(
|
| 261 |
+
gk + (bos + last_idx) * H * K + i_h * K + o_k4,
|
| 262 |
+
mask=(o_k4 < K),
|
| 263 |
+
other=0.0,
|
| 264 |
+
)
|
| 265 |
+
if USE_EXP2:
|
| 266 |
+
b_h4 *= exp2(b_gk_last4)[None, :]
|
| 267 |
+
else:
|
| 268 |
+
b_h4 *= exp(b_gk_last4)[None, :]
|
| 269 |
+
b_v = b_v.to(k.dtype.element_ty)
|
| 270 |
+
|
| 271 |
+
p_k = tl.make_block_ptr(
|
| 272 |
+
k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1)
|
| 273 |
+
)
|
| 274 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 275 |
+
b_h1 += tl.trans(tl.dot(b_k, b_v))
|
| 276 |
+
if K > 64:
|
| 277 |
+
p_k = tl.make_block_ptr(
|
| 278 |
+
k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1)
|
| 279 |
+
)
|
| 280 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 281 |
+
b_h2 += tl.trans(tl.dot(b_k, b_v))
|
| 282 |
+
if K > 128:
|
| 283 |
+
p_k = tl.make_block_ptr(
|
| 284 |
+
k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1)
|
| 285 |
+
)
|
| 286 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 287 |
+
b_h3 += tl.trans(tl.dot(b_k, b_v))
|
| 288 |
+
if K > 192:
|
| 289 |
+
p_k = tl.make_block_ptr(
|
| 290 |
+
k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1)
|
| 291 |
+
)
|
| 292 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 293 |
+
b_h4 += tl.trans(tl.dot(b_k, b_v))
|
| 294 |
+
|
| 295 |
+
# epilogue
|
| 296 |
+
if INPLACE_UPDATE and valid_state:
|
| 297 |
+
p_ht = tl.make_block_ptr(ht, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0))
|
| 298 |
+
tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
|
| 299 |
+
if K > 64:
|
| 300 |
+
p_ht = tl.make_block_ptr(
|
| 301 |
+
ht, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0)
|
| 302 |
+
)
|
| 303 |
+
tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
|
| 304 |
+
if K > 128:
|
| 305 |
+
p_ht = tl.make_block_ptr(
|
| 306 |
+
ht, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0)
|
| 307 |
+
)
|
| 308 |
+
tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
|
| 309 |
+
if K > 192:
|
| 310 |
+
p_ht = tl.make_block_ptr(
|
| 311 |
+
ht, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0)
|
| 312 |
+
)
|
| 313 |
+
tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1))
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
def chunk_gated_delta_rule_fwd_h(
|
| 317 |
+
k: torch.Tensor,
|
| 318 |
+
w: torch.Tensor,
|
| 319 |
+
u: torch.Tensor,
|
| 320 |
+
g: Optional[torch.Tensor] = None,
|
| 321 |
+
gk: Optional[torch.Tensor] = None,
|
| 322 |
+
initial_state: Optional[torch.Tensor] = None,
|
| 323 |
+
initial_state_indices: Optional[torch.Tensor] = None,
|
| 324 |
+
save_new_value: bool = True,
|
| 325 |
+
cu_seqlens: Optional[torch.LongTensor] = None,
|
| 326 |
+
chunk_indices: Optional[torch.LongTensor] = None,
|
| 327 |
+
use_exp2: bool = False,
|
| 328 |
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 329 |
+
assert not (
|
| 330 |
+
use_exp2 and g is not None
|
| 331 |
+
), "use_exp2 covers only the per-channel gk path; scalar g stays natural-exp"
|
| 332 |
+
B, T, Hg, K, V = *k.shape, u.shape[-1]
|
| 333 |
+
H = u.shape[-2]
|
| 334 |
+
BT = CHUNK_SIZE
|
| 335 |
+
|
| 336 |
+
if chunk_indices is None and cu_seqlens is not None:
|
| 337 |
+
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
| 338 |
+
# N: the actual number of sequences in the batch with either equal or variable lengths
|
| 339 |
+
if cu_seqlens is None:
|
| 340 |
+
N, NT, chunk_offsets = B, triton.cdiv(T, BT), None
|
| 341 |
+
else:
|
| 342 |
+
N, NT, chunk_offsets = (
|
| 343 |
+
len(cu_seqlens) - 1,
|
| 344 |
+
len(chunk_indices),
|
| 345 |
+
prepare_chunk_offsets(cu_seqlens, BT),
|
| 346 |
+
)
|
| 347 |
+
assert K <= 256, "current kernel does not support head dimension larger than 256."
|
| 348 |
+
|
| 349 |
+
h = k.new_empty(B, NT, H, V, K)
|
| 350 |
+
|
| 351 |
+
v_new = torch.empty_like(u) if save_new_value else None
|
| 352 |
+
|
| 353 |
+
def grid(meta: Any) -> Any:
|
| 354 |
+
return (triton.cdiv(V, meta["BV"]), N * H)
|
| 355 |
+
|
| 356 |
+
chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid](
|
| 357 |
+
k=k,
|
| 358 |
+
v=u,
|
| 359 |
+
w=w,
|
| 360 |
+
v_new=v_new,
|
| 361 |
+
g=g,
|
| 362 |
+
gk=gk,
|
| 363 |
+
h=h,
|
| 364 |
+
initial_state=initial_state,
|
| 365 |
+
initial_state_indices=initial_state_indices,
|
| 366 |
+
# Envelope-strided state pools (page-major / unified memory) have a
|
| 367 |
+
# per-slot pitch != H*V*K; contiguous pools pass exactly H*V*K.
|
| 368 |
+
stride_init_state=(initial_state.stride(0) if initial_state is not None else 0),
|
| 369 |
+
cu_seqlens=cu_seqlens,
|
| 370 |
+
chunk_offsets=chunk_offsets,
|
| 371 |
+
T=T,
|
| 372 |
+
H=H,
|
| 373 |
+
Hg=Hg,
|
| 374 |
+
K=K,
|
| 375 |
+
V=V,
|
| 376 |
+
BT=BT,
|
| 377 |
+
USE_G=g is not None,
|
| 378 |
+
USE_GK=gk is not None,
|
| 379 |
+
USE_INITIAL_STATE=initial_state is not None,
|
| 380 |
+
INPLACE_UPDATE=True,
|
| 381 |
+
SAVE_NEW_VALUE=v_new is not None,
|
| 382 |
+
IS_VARLEN=cu_seqlens is not None,
|
| 383 |
+
NT_BUCKET=(0 if NT <= 32 else (1 if NT <= 128 else 2)),
|
| 384 |
+
USE_EXP2=use_exp2,
|
| 385 |
+
)
|
| 386 |
+
return h, v_new
|
clairic_runtime/gdn/chunk_fwd.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/chunk_fwd.py
|
| 2 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 3 |
+
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .index import prepare_chunk_indices
|
| 10 |
+
from .op import safe_exp
|
| 11 |
+
from .utils import (
|
| 12 |
+
autotune_cache_kwargs,
|
| 13 |
+
is_tf32_supported,
|
| 14 |
+
)
|
| 15 |
+
from .wy_fast import recompute_w_u_fwd
|
| 16 |
+
|
| 17 |
+
# TF32 for the block-merge dot products (16x16 matmuls) is safe and ~2x faster on SM90.
|
| 18 |
+
# The numerically sensitive forward-substitution uses scalar ops, not tl.dot.
|
| 19 |
+
if is_tf32_supported:
|
| 20 |
+
_MERGE_DOT_PRECISION = tl.constexpr("tf32")
|
| 21 |
+
else:
|
| 22 |
+
_MERGE_DOT_PRECISION = tl.constexpr("ieee")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@triton.heuristics(
|
| 26 |
+
{
|
| 27 |
+
"USE_G": lambda args: args["g"] is not None,
|
| 28 |
+
"IS_VARLEN": lambda args: args["cu_seqlens"] is not None,
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
@triton.autotune(
|
| 32 |
+
configs=[
|
| 33 |
+
triton.Config({"BK": BK}, num_warps=num_warps)
|
| 34 |
+
for BK in [32, 64]
|
| 35 |
+
for num_warps in [1, 2, 4]
|
| 36 |
+
],
|
| 37 |
+
key=["H", "Hg", "K", "BC"],
|
| 38 |
+
**autotune_cache_kwargs,
|
| 39 |
+
)
|
| 40 |
+
@triton.jit(do_not_specialize=["T"])
|
| 41 |
+
def chunk_gated_delta_rule_fwd_kkt_solve_kernel(
|
| 42 |
+
k: tl.tensor,
|
| 43 |
+
g: tl.tensor,
|
| 44 |
+
beta: tl.tensor,
|
| 45 |
+
A: tl.tensor,
|
| 46 |
+
cu_seqlens: tl.tensor,
|
| 47 |
+
chunk_indices: tl.tensor,
|
| 48 |
+
T: tl.int32,
|
| 49 |
+
H: tl.constexpr,
|
| 50 |
+
Hg: tl.constexpr,
|
| 51 |
+
K: tl.constexpr,
|
| 52 |
+
BT: tl.constexpr,
|
| 53 |
+
BC: tl.constexpr,
|
| 54 |
+
BK: tl.constexpr,
|
| 55 |
+
USE_G: tl.constexpr,
|
| 56 |
+
IS_VARLEN: tl.constexpr,
|
| 57 |
+
) -> None:
|
| 58 |
+
"""
|
| 59 |
+
Fused kernel: compute beta * K @ K^T (lower triangular) + solve_tril (I+A)^{-1} in one pass.
|
| 60 |
+
|
| 61 |
+
This kernel fuses chunk_scaled_dot_kkt_fwd and solve_tril into a single kernel,
|
| 62 |
+
avoiding the HBM round-trip for the intermediate A matrix.
|
| 63 |
+
|
| 64 |
+
Steps:
|
| 65 |
+
1. Compute all 10 lower-triangular [BC, BC] blocks of beta * K @ K^T in registers
|
| 66 |
+
2. Apply gate and beta scaling
|
| 67 |
+
3. Forward substitution on diagonal blocks
|
| 68 |
+
4. Block merge to get full (I+A)^{-1}
|
| 69 |
+
5. Write result to A (output)
|
| 70 |
+
"""
|
| 71 |
+
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
| 72 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 73 |
+
|
| 74 |
+
if IS_VARLEN:
|
| 75 |
+
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
|
| 76 |
+
chunk_indices + i_t * 2 + 1
|
| 77 |
+
).to(tl.int32)
|
| 78 |
+
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
|
| 79 |
+
cu_seqlens + i_n + 1
|
| 80 |
+
).to(tl.int32)
|
| 81 |
+
T = eos - bos
|
| 82 |
+
else:
|
| 83 |
+
bos, eos = i_b * T, i_b * T + T
|
| 84 |
+
|
| 85 |
+
if i_t * BT >= T:
|
| 86 |
+
return
|
| 87 |
+
|
| 88 |
+
i_tc0 = i_t * BT
|
| 89 |
+
i_tc1 = i_t * BT + BC
|
| 90 |
+
i_tc2 = i_t * BT + 2 * BC
|
| 91 |
+
i_tc3 = i_t * BT + 3 * BC
|
| 92 |
+
|
| 93 |
+
k += (bos * Hg + i_h // (H // Hg)) * K
|
| 94 |
+
A += (bos * H + i_h) * BT
|
| 95 |
+
|
| 96 |
+
o_i = tl.arange(0, BC)
|
| 97 |
+
m_tc0 = (i_tc0 + o_i) < T
|
| 98 |
+
m_tc1 = (i_tc1 + o_i) < T
|
| 99 |
+
m_tc2 = (i_tc2 + o_i) < T
|
| 100 |
+
m_tc3 = (i_tc3 + o_i) < T
|
| 101 |
+
|
| 102 |
+
# load beta for each sub-chunk
|
| 103 |
+
p_b0 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc0,), (BC,), (0,))
|
| 104 |
+
p_b1 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc1,), (BC,), (0,))
|
| 105 |
+
p_b2 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc2,), (BC,), (0,))
|
| 106 |
+
p_b3 = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_tc3,), (BC,), (0,))
|
| 107 |
+
b_b0 = tl.load(p_b0, boundary_check=(0,)).to(tl.float32)
|
| 108 |
+
b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32)
|
| 109 |
+
b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32)
|
| 110 |
+
b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32)
|
| 111 |
+
|
| 112 |
+
# load gate if used
|
| 113 |
+
if USE_G:
|
| 114 |
+
p_g0 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc0,), (BC,), (0,))
|
| 115 |
+
p_g1 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc1,), (BC,), (0,))
|
| 116 |
+
p_g2 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc2,), (BC,), (0,))
|
| 117 |
+
p_g3 = tl.make_block_ptr(g + bos * H + i_h, (T,), (H,), (i_tc3,), (BC,), (0,))
|
| 118 |
+
|
| 119 |
+
b_g0 = tl.load(p_g0, boundary_check=(0,)).to(tl.float32)
|
| 120 |
+
b_g1 = tl.load(p_g1, boundary_check=(0,)).to(tl.float32)
|
| 121 |
+
b_g2 = tl.load(p_g2, boundary_check=(0,)).to(tl.float32)
|
| 122 |
+
b_g3 = tl.load(p_g3, boundary_check=(0,)).to(tl.float32)
|
| 123 |
+
|
| 124 |
+
############################################################################
|
| 125 |
+
# Step 1: compute all 10 lower-triangular [BC, BC] blocks of K @ K^T
|
| 126 |
+
############################################################################
|
| 127 |
+
|
| 128 |
+
# 4 diagonal blocks
|
| 129 |
+
b_A00 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 130 |
+
b_A11 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 131 |
+
b_A22 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 132 |
+
b_A33 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 133 |
+
|
| 134 |
+
# 6 off-diagonal blocks
|
| 135 |
+
b_A10 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 136 |
+
b_A20 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 137 |
+
b_A21 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 138 |
+
b_A30 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 139 |
+
b_A31 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 140 |
+
b_A32 = tl.zeros([BC, BC], dtype=tl.float32)
|
| 141 |
+
|
| 142 |
+
for i_k in range(tl.cdiv(K, BK)):
|
| 143 |
+
p_k0 = tl.make_block_ptr(
|
| 144 |
+
k, (T, K), (Hg * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0)
|
| 145 |
+
)
|
| 146 |
+
b_k0 = tl.load(p_k0, boundary_check=(0, 1))
|
| 147 |
+
# diagonal block 0
|
| 148 |
+
b_A00 += tl.dot(b_k0, tl.trans(b_k0))
|
| 149 |
+
|
| 150 |
+
if i_tc1 < T:
|
| 151 |
+
p_k1 = tl.make_block_ptr(
|
| 152 |
+
k, (T, K), (Hg * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0)
|
| 153 |
+
)
|
| 154 |
+
b_k1 = tl.load(p_k1, boundary_check=(0, 1))
|
| 155 |
+
# diagonal block 1
|
| 156 |
+
b_A11 += tl.dot(b_k1, tl.trans(b_k1))
|
| 157 |
+
# off-diagonal (1,0)
|
| 158 |
+
b_A10 += tl.dot(b_k1, tl.trans(b_k0))
|
| 159 |
+
|
| 160 |
+
if i_tc2 < T:
|
| 161 |
+
p_k2 = tl.make_block_ptr(
|
| 162 |
+
k, (T, K), (Hg * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0)
|
| 163 |
+
)
|
| 164 |
+
b_k2 = tl.load(p_k2, boundary_check=(0, 1))
|
| 165 |
+
# diagonal block 2
|
| 166 |
+
b_A22 += tl.dot(b_k2, tl.trans(b_k2))
|
| 167 |
+
# off-diagonal (2,0), (2,1)
|
| 168 |
+
b_A20 += tl.dot(b_k2, tl.trans(b_k0))
|
| 169 |
+
b_A21 += tl.dot(b_k2, tl.trans(b_k1))
|
| 170 |
+
|
| 171 |
+
if i_tc3 < T:
|
| 172 |
+
p_k3 = tl.make_block_ptr(
|
| 173 |
+
k, (T, K), (Hg * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0)
|
| 174 |
+
)
|
| 175 |
+
b_k3 = tl.load(p_k3, boundary_check=(0, 1))
|
| 176 |
+
# diagonal block 3
|
| 177 |
+
b_A33 += tl.dot(b_k3, tl.trans(b_k3))
|
| 178 |
+
# off-diagonal (3,0), (3,1), (3,2)
|
| 179 |
+
b_A30 += tl.dot(b_k3, tl.trans(b_k0))
|
| 180 |
+
b_A31 += tl.dot(b_k3, tl.trans(b_k1))
|
| 181 |
+
b_A32 += tl.dot(b_k3, tl.trans(b_k2))
|
| 182 |
+
|
| 183 |
+
############################################################################
|
| 184 |
+
# Step 2: apply gate and beta scaling
|
| 185 |
+
############################################################################
|
| 186 |
+
|
| 187 |
+
if USE_G:
|
| 188 |
+
# diagonal blocks: g_diff = g_i - g_j within sub-chunk
|
| 189 |
+
b_A00 *= safe_exp(b_g0[:, None] - b_g0[None, :])
|
| 190 |
+
b_A11 *= safe_exp(b_g1[:, None] - b_g1[None, :])
|
| 191 |
+
b_A22 *= safe_exp(b_g2[:, None] - b_g2[None, :])
|
| 192 |
+
b_A33 *= safe_exp(b_g3[:, None] - b_g3[None, :])
|
| 193 |
+
|
| 194 |
+
# off-diagonal blocks: g_diff = g_row - g_col (cross sub-chunk)
|
| 195 |
+
b_A10 *= safe_exp(b_g1[:, None] - b_g0[None, :])
|
| 196 |
+
b_A20 *= safe_exp(b_g2[:, None] - b_g0[None, :])
|
| 197 |
+
b_A21 *= safe_exp(b_g2[:, None] - b_g1[None, :])
|
| 198 |
+
b_A30 *= safe_exp(b_g3[:, None] - b_g0[None, :])
|
| 199 |
+
b_A31 *= safe_exp(b_g3[:, None] - b_g1[None, :])
|
| 200 |
+
b_A32 *= safe_exp(b_g3[:, None] - b_g2[None, :])
|
| 201 |
+
|
| 202 |
+
# apply beta to row dimension and mask
|
| 203 |
+
m_d = o_i[:, None] > o_i[None, :]
|
| 204 |
+
m_I = o_i[:, None] == o_i[None, :]
|
| 205 |
+
|
| 206 |
+
# diagonal blocks: strictly lower triangular within sub-chunk, scaled by beta
|
| 207 |
+
b_A00 = (
|
| 208 |
+
tl.where(m_d & (m_tc0[:, None] & m_tc0[None, :]), b_A00, 0.0) * b_b0[:, None]
|
| 209 |
+
)
|
| 210 |
+
b_A11 = (
|
| 211 |
+
tl.where(m_d & (m_tc1[:, None] & m_tc1[None, :]), b_A11, 0.0) * b_b1[:, None]
|
| 212 |
+
)
|
| 213 |
+
b_A22 = (
|
| 214 |
+
tl.where(m_d & (m_tc2[:, None] & m_tc2[None, :]), b_A22, 0.0) * b_b2[:, None]
|
| 215 |
+
)
|
| 216 |
+
b_A33 = (
|
| 217 |
+
tl.where(m_d & (m_tc3[:, None] & m_tc3[None, :]), b_A33, 0.0) * b_b3[:, None]
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
# off-diagonal blocks: full block, scaled by beta
|
| 221 |
+
b_A10 = b_A10 * b_b1[:, None]
|
| 222 |
+
b_A20 = b_A20 * b_b2[:, None]
|
| 223 |
+
b_A21 = b_A21 * b_b2[:, None]
|
| 224 |
+
b_A30 = b_A30 * b_b3[:, None]
|
| 225 |
+
b_A31 = b_A31 * b_b3[:, None]
|
| 226 |
+
b_A32 = b_A32 * b_b3[:, None]
|
| 227 |
+
|
| 228 |
+
############################################################################
|
| 229 |
+
# Step 3: forward substitution on diagonal blocks -> (I + A_diag)^{-1}
|
| 230 |
+
#
|
| 231 |
+
# Same algorithm as solve_tril, but rows are extracted from in-register
|
| 232 |
+
# [BC, BC] tensor via tl.sum(tl.where(mask, tensor, 0), 0) instead of
|
| 233 |
+
# tl.load from HBM.
|
| 234 |
+
############################################################################
|
| 235 |
+
|
| 236 |
+
b_Ai00 = -b_A00
|
| 237 |
+
b_Ai11 = -b_A11
|
| 238 |
+
b_Ai22 = -b_A22
|
| 239 |
+
b_Ai33 = -b_A33
|
| 240 |
+
|
| 241 |
+
for i in range(2, min(BC, T - i_tc0)):
|
| 242 |
+
b_a00 = tl.sum(tl.where((o_i == i)[:, None], -b_A00, 0.0), 0)
|
| 243 |
+
b_a00 = tl.where(o_i < i, b_a00, 0.0)
|
| 244 |
+
b_a00 = b_a00 + tl.sum(b_a00[:, None] * b_Ai00, 0)
|
| 245 |
+
b_Ai00 = tl.where((o_i == i)[:, None], b_a00, b_Ai00)
|
| 246 |
+
for i in range(2, min(BC, T - i_tc1)):
|
| 247 |
+
b_a11 = tl.sum(tl.where((o_i == i)[:, None], -b_A11, 0.0), 0)
|
| 248 |
+
b_a11 = tl.where(o_i < i, b_a11, 0.0)
|
| 249 |
+
b_a11 = b_a11 + tl.sum(b_a11[:, None] * b_Ai11, 0)
|
| 250 |
+
b_Ai11 = tl.where((o_i == i)[:, None], b_a11, b_Ai11)
|
| 251 |
+
for i in range(2, min(BC, T - i_tc2)):
|
| 252 |
+
b_a22 = tl.sum(tl.where((o_i == i)[:, None], -b_A22, 0.0), 0)
|
| 253 |
+
b_a22 = tl.where(o_i < i, b_a22, 0.0)
|
| 254 |
+
b_a22 = b_a22 + tl.sum(b_a22[:, None] * b_Ai22, 0)
|
| 255 |
+
b_Ai22 = tl.where((o_i == i)[:, None], b_a22, b_Ai22)
|
| 256 |
+
for i in range(2, min(BC, T - i_tc3)):
|
| 257 |
+
b_a33 = tl.sum(tl.where((o_i == i)[:, None], -b_A33, 0.0), 0)
|
| 258 |
+
b_a33 = tl.where(o_i < i, b_a33, 0.0)
|
| 259 |
+
b_a33 = b_a33 + tl.sum(b_a33[:, None] * b_Ai33, 0)
|
| 260 |
+
b_Ai33 = tl.where((o_i == i)[:, None], b_a33, b_Ai33)
|
| 261 |
+
|
| 262 |
+
b_Ai00 += m_I
|
| 263 |
+
b_Ai11 += m_I
|
| 264 |
+
b_Ai22 += m_I
|
| 265 |
+
b_Ai33 += m_I
|
| 266 |
+
|
| 267 |
+
############################################################################
|
| 268 |
+
# Step 4: block merge -> full (I + A)^{-1}
|
| 269 |
+
############################################################################
|
| 270 |
+
|
| 271 |
+
b_Ai10 = -tl.dot(
|
| 272 |
+
tl.dot(b_Ai11, b_A10, input_precision=_MERGE_DOT_PRECISION),
|
| 273 |
+
b_Ai00,
|
| 274 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 275 |
+
)
|
| 276 |
+
b_Ai21 = -tl.dot(
|
| 277 |
+
tl.dot(b_Ai22, b_A21, input_precision=_MERGE_DOT_PRECISION),
|
| 278 |
+
b_Ai11,
|
| 279 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 280 |
+
)
|
| 281 |
+
b_Ai32 = -tl.dot(
|
| 282 |
+
tl.dot(b_Ai33, b_A32, input_precision=_MERGE_DOT_PRECISION),
|
| 283 |
+
b_Ai22,
|
| 284 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 285 |
+
)
|
| 286 |
+
|
| 287 |
+
b_Ai20 = -tl.dot(
|
| 288 |
+
b_Ai22,
|
| 289 |
+
tl.dot(b_A20, b_Ai00, input_precision=_MERGE_DOT_PRECISION)
|
| 290 |
+
+ tl.dot(b_A21, b_Ai10, input_precision=_MERGE_DOT_PRECISION),
|
| 291 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 292 |
+
)
|
| 293 |
+
b_Ai31 = -tl.dot(
|
| 294 |
+
b_Ai33,
|
| 295 |
+
tl.dot(b_A31, b_Ai11, input_precision=_MERGE_DOT_PRECISION)
|
| 296 |
+
+ tl.dot(b_A32, b_Ai21, input_precision=_MERGE_DOT_PRECISION),
|
| 297 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 298 |
+
)
|
| 299 |
+
b_Ai30 = -tl.dot(
|
| 300 |
+
b_Ai33,
|
| 301 |
+
tl.dot(b_A30, b_Ai00, input_precision=_MERGE_DOT_PRECISION)
|
| 302 |
+
+ tl.dot(b_A31, b_Ai10, input_precision=_MERGE_DOT_PRECISION)
|
| 303 |
+
+ tl.dot(b_A32, b_Ai20, input_precision=_MERGE_DOT_PRECISION),
|
| 304 |
+
input_precision=_MERGE_DOT_PRECISION,
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
############################################################################
|
| 308 |
+
# Step 5: store full (I + A)^{-1} to output A
|
| 309 |
+
############################################################################
|
| 310 |
+
|
| 311 |
+
p_A00 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc0, 0), (BC, BC), (1, 0))
|
| 312 |
+
p_A10 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc1, 0), (BC, BC), (1, 0))
|
| 313 |
+
p_A11 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc1, BC), (BC, BC), (1, 0))
|
| 314 |
+
p_A20 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc2, 0), (BC, BC), (1, 0))
|
| 315 |
+
p_A21 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc2, BC), (BC, BC), (1, 0))
|
| 316 |
+
p_A22 = tl.make_block_ptr(
|
| 317 |
+
A, (T, BT), (H * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0)
|
| 318 |
+
)
|
| 319 |
+
p_A30 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc3, 0), (BC, BC), (1, 0))
|
| 320 |
+
p_A31 = tl.make_block_ptr(A, (T, BT), (H * BT, 1), (i_tc3, BC), (BC, BC), (1, 0))
|
| 321 |
+
p_A32 = tl.make_block_ptr(
|
| 322 |
+
A, (T, BT), (H * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0)
|
| 323 |
+
)
|
| 324 |
+
p_A33 = tl.make_block_ptr(
|
| 325 |
+
A, (T, BT), (H * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0)
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
tl.store(p_A00, b_Ai00.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 329 |
+
tl.store(p_A10, b_Ai10.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 330 |
+
tl.store(p_A11, b_Ai11.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 331 |
+
tl.store(p_A20, b_Ai20.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 332 |
+
tl.store(p_A21, b_Ai21.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 333 |
+
tl.store(p_A22, b_Ai22.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 334 |
+
tl.store(p_A30, b_Ai30.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 335 |
+
tl.store(p_A31, b_Ai31.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 336 |
+
tl.store(p_A32, b_Ai32.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 337 |
+
tl.store(p_A33, b_Ai33.to(A.dtype.element_ty), boundary_check=(0, 1))
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def chunk_gated_delta_rule_fwd_intra(
|
| 341 |
+
k: torch.Tensor,
|
| 342 |
+
v: torch.Tensor,
|
| 343 |
+
g: torch.Tensor | None = None,
|
| 344 |
+
beta: torch.Tensor | None = None,
|
| 345 |
+
cu_seqlens: torch.LongTensor | None = None,
|
| 346 |
+
chunk_size: int = 64,
|
| 347 |
+
chunk_indices: torch.LongTensor | None = None,
|
| 348 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 349 |
+
r"""
|
| 350 |
+
GDN intra-chunk forward: fused kkt + solve_tril + recompute_w_u.
|
| 351 |
+
|
| 352 |
+
Equivalent to:
|
| 353 |
+
A = chunk_scaled_dot_kkt_fwd(k, g, beta, ...) # kernel 1
|
| 354 |
+
A = solve_tril(A, ...) # kernel 2
|
| 355 |
+
w, u = recompute_w_u_fwd(k, v, beta, A, g, ...) # kernel 3
|
| 356 |
+
|
| 357 |
+
Fuses kernels 1+2 into a single kernel, reducing from 3 to 2 kernel launches
|
| 358 |
+
and eliminating the HBM round-trip for the intermediate A matrix.
|
| 359 |
+
|
| 360 |
+
Args:
|
| 361 |
+
k (torch.Tensor):
|
| 362 |
+
The key tensor of shape `[B, T, H, K]`.
|
| 363 |
+
v (torch.Tensor):
|
| 364 |
+
The value tensor of shape `[B, T, H, V]`.
|
| 365 |
+
g (torch.Tensor):
|
| 366 |
+
The cumulative sum of the gate tensor of shape `[B, T, H]`. Default: `None`.
|
| 367 |
+
beta (torch.Tensor):
|
| 368 |
+
The beta tensor of shape `[B, T, H]`.
|
| 369 |
+
cu_seqlens (torch.LongTensor):
|
| 370 |
+
The cumulative sequence lengths. Default: `None`.
|
| 371 |
+
chunk_size (int):
|
| 372 |
+
The chunk size. Default: 64.
|
| 373 |
+
chunk_indices (torch.LongTensor):
|
| 374 |
+
Precomputed chunk indices. Default: `None`.
|
| 375 |
+
|
| 376 |
+
Returns:
|
| 377 |
+
w (torch.Tensor): shape `[B, T, H, K]`
|
| 378 |
+
u (torch.Tensor): shape `[B, T, H, V]`
|
| 379 |
+
A (torch.Tensor): shape `[B, T, H, BT]`, the solved (I+A)^{-1} matrix
|
| 380 |
+
"""
|
| 381 |
+
B, T, Hg, K = k.shape
|
| 382 |
+
H = beta.shape[-1]
|
| 383 |
+
BT = chunk_size
|
| 384 |
+
BC = 16
|
| 385 |
+
|
| 386 |
+
if chunk_indices is None and cu_seqlens is not None:
|
| 387 |
+
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
| 388 |
+
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
| 389 |
+
|
| 390 |
+
# Step 1: fused kkt + solve_tril
|
| 391 |
+
A = torch.zeros(B, T, H, BT, device=k.device, dtype=k.dtype)
|
| 392 |
+
chunk_gated_delta_rule_fwd_kkt_solve_kernel[(NT, B * H)](
|
| 393 |
+
k=k,
|
| 394 |
+
g=g,
|
| 395 |
+
beta=beta,
|
| 396 |
+
A=A,
|
| 397 |
+
cu_seqlens=cu_seqlens,
|
| 398 |
+
chunk_indices=chunk_indices,
|
| 399 |
+
T=T,
|
| 400 |
+
H=H,
|
| 401 |
+
Hg=Hg,
|
| 402 |
+
K=K,
|
| 403 |
+
BT=BT,
|
| 404 |
+
BC=BC,
|
| 405 |
+
)
|
| 406 |
+
|
| 407 |
+
# Step 2: recompute_w_u
|
| 408 |
+
w, u = recompute_w_u_fwd(
|
| 409 |
+
k=k,
|
| 410 |
+
v=v,
|
| 411 |
+
beta=beta,
|
| 412 |
+
A=A,
|
| 413 |
+
g_cumsum=g,
|
| 414 |
+
cu_seqlens=cu_seqlens,
|
| 415 |
+
chunk_indices=chunk_indices,
|
| 416 |
+
)
|
| 417 |
+
return w, u, A
|
clairic_runtime/gdn/chunk_o.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_o.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import triton
|
| 10 |
+
import triton.language as tl
|
| 11 |
+
|
| 12 |
+
from .index import prepare_chunk_indices
|
| 13 |
+
from .op import exp, safe_exp
|
| 14 |
+
from .utils import check_shared_mem, is_nvidia_hopper
|
| 15 |
+
|
| 16 |
+
BKV_LIST = [64, 128] if check_shared_mem() else [32, 64]
|
| 17 |
+
NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
# @triton.autotune(
|
| 21 |
+
# configs=[
|
| 22 |
+
# triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages)
|
| 23 |
+
# for BK in BKV_LIST
|
| 24 |
+
# for BV in BKV_LIST
|
| 25 |
+
# for num_warps in NUM_WARPS
|
| 26 |
+
# for num_stages in [2, 3, 4]
|
| 27 |
+
# ],
|
| 28 |
+
# key=["H", "K", "V", "BT"],
|
| 29 |
+
# )
|
| 30 |
+
@triton.jit(do_not_specialize=["T"])
|
| 31 |
+
def chunk_fwd_kernel_o(
|
| 32 |
+
q: tl.tensor,
|
| 33 |
+
k: tl.tensor,
|
| 34 |
+
v: tl.tensor,
|
| 35 |
+
h: tl.tensor,
|
| 36 |
+
g: tl.tensor,
|
| 37 |
+
o: tl.tensor,
|
| 38 |
+
cu_seqlens: tl.tensor,
|
| 39 |
+
chunk_indices: tl.tensor,
|
| 40 |
+
scale: tl.float32,
|
| 41 |
+
T: tl.int32,
|
| 42 |
+
H: tl.constexpr,
|
| 43 |
+
Hg: tl.constexpr,
|
| 44 |
+
K: tl.constexpr,
|
| 45 |
+
V: tl.constexpr,
|
| 46 |
+
BT: tl.constexpr,
|
| 47 |
+
BK: tl.constexpr,
|
| 48 |
+
BV: tl.constexpr,
|
| 49 |
+
USE_G: tl.constexpr,
|
| 50 |
+
IS_VARLEN: tl.constexpr,
|
| 51 |
+
) -> None:
|
| 52 |
+
i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 53 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 54 |
+
|
| 55 |
+
if IS_VARLEN:
|
| 56 |
+
i_tg = i_t
|
| 57 |
+
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
|
| 58 |
+
chunk_indices + i_t * 2 + 1
|
| 59 |
+
).to(tl.int32)
|
| 60 |
+
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
|
| 61 |
+
cu_seqlens + i_n + 1
|
| 62 |
+
).to(tl.int32)
|
| 63 |
+
T = eos - bos
|
| 64 |
+
NT = tl.cdiv(T, BT)
|
| 65 |
+
else:
|
| 66 |
+
NT = tl.cdiv(T, BT)
|
| 67 |
+
i_tg = i_b * NT + i_t
|
| 68 |
+
bos, eos = i_b * T, i_b * T + T
|
| 69 |
+
|
| 70 |
+
# offset calculation
|
| 71 |
+
q += (bos * Hg + i_h // (H // Hg)) * K
|
| 72 |
+
k += (bos * Hg + i_h // (H // Hg)) * K
|
| 73 |
+
v += (bos * H + i_h) * V
|
| 74 |
+
o += (bos * H + i_h) * V
|
| 75 |
+
h += (i_tg * H + i_h).to(tl.int64) * V * K
|
| 76 |
+
|
| 77 |
+
b_o = tl.zeros([BT, BV], dtype=tl.float32)
|
| 78 |
+
b_A = tl.zeros([BT, BT], dtype=tl.float32)
|
| 79 |
+
|
| 80 |
+
for i_k in range(tl.cdiv(K, BK)):
|
| 81 |
+
p_q = tl.make_block_ptr(
|
| 82 |
+
q, (T, K), (Hg * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0)
|
| 83 |
+
)
|
| 84 |
+
p_k = tl.make_block_ptr(
|
| 85 |
+
k, (K, T), (1, Hg * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1)
|
| 86 |
+
)
|
| 87 |
+
p_h = tl.make_block_ptr(
|
| 88 |
+
h, (V, K), (K, 1), (i_v * BV, i_k * BK), (BV, BK), (1, 0)
|
| 89 |
+
)
|
| 90 |
+
# [BT, BK]
|
| 91 |
+
b_q = tl.load(p_q, boundary_check=(0, 1))
|
| 92 |
+
# [BK, BT]
|
| 93 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 94 |
+
# [BV, BK]
|
| 95 |
+
b_h = tl.load(p_h, boundary_check=(0, 1))
|
| 96 |
+
|
| 97 |
+
# [BT, BK] @ [BK, BV] -> [BT, BV]
|
| 98 |
+
b_o += tl.dot(b_q, tl.trans(b_h))
|
| 99 |
+
# [BT, BK] @ [BK, BT] -> [BT, BT]
|
| 100 |
+
b_A += tl.dot(b_q, b_k)
|
| 101 |
+
|
| 102 |
+
if USE_G:
|
| 103 |
+
g += bos * H + i_h
|
| 104 |
+
p_g = tl.make_block_ptr(g, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
| 105 |
+
b_g = tl.load(p_g, boundary_check=(0,))
|
| 106 |
+
b_o = b_o * exp(b_g)[:, None]
|
| 107 |
+
b_A = b_A * safe_exp(b_g[:, None] - b_g[None, :])
|
| 108 |
+
|
| 109 |
+
o_i = tl.arange(0, BT)
|
| 110 |
+
m_A = o_i[:, None] >= o_i[None, :]
|
| 111 |
+
b_A = tl.where(m_A, b_A, 0)
|
| 112 |
+
|
| 113 |
+
p_v = tl.make_block_ptr(
|
| 114 |
+
v, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
|
| 115 |
+
)
|
| 116 |
+
p_o = tl.make_block_ptr(
|
| 117 |
+
o, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0)
|
| 118 |
+
)
|
| 119 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 120 |
+
|
| 121 |
+
# to fix mma -> mma layout conversion
|
| 122 |
+
# already solved by triton v3.2 or higher
|
| 123 |
+
b_o = b_o * scale + tl.dot(b_A.to(b_v.dtype), b_v) * scale
|
| 124 |
+
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1))
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def chunk_fwd_o(
|
| 128 |
+
q: torch.Tensor,
|
| 129 |
+
k: torch.Tensor,
|
| 130 |
+
v: torch.Tensor,
|
| 131 |
+
h: torch.Tensor,
|
| 132 |
+
g: Optional[torch.Tensor] = None, # cumsum of log decay
|
| 133 |
+
scale: Optional[float] = None,
|
| 134 |
+
cu_seqlens: Optional[torch.LongTensor] = None,
|
| 135 |
+
chunk_size: int = 64,
|
| 136 |
+
) -> torch.Tensor:
|
| 137 |
+
B, T, Hg, K, V = *q.shape, v.shape[-1]
|
| 138 |
+
H = v.shape[-2]
|
| 139 |
+
BT = min(chunk_size, max(16, triton.next_power_of_2(T)))
|
| 140 |
+
chunk_indices = (
|
| 141 |
+
prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None
|
| 142 |
+
)
|
| 143 |
+
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
| 144 |
+
if scale is None:
|
| 145 |
+
scale = k.shape[-1] ** -0.5
|
| 146 |
+
|
| 147 |
+
o = torch.zeros_like(v)
|
| 148 |
+
|
| 149 |
+
def grid(meta: dict[str, int]) -> tuple[int, int, int]:
|
| 150 |
+
return (triton.cdiv(V, meta["BV"]), NT, B * H)
|
| 151 |
+
|
| 152 |
+
chunk_fwd_kernel_o[grid](
|
| 153 |
+
q,
|
| 154 |
+
k,
|
| 155 |
+
v,
|
| 156 |
+
h,
|
| 157 |
+
g,
|
| 158 |
+
o,
|
| 159 |
+
cu_seqlens,
|
| 160 |
+
chunk_indices,
|
| 161 |
+
scale,
|
| 162 |
+
T=T,
|
| 163 |
+
H=H,
|
| 164 |
+
Hg=Hg,
|
| 165 |
+
K=K,
|
| 166 |
+
V=V,
|
| 167 |
+
BT=BT,
|
| 168 |
+
BK=128,
|
| 169 |
+
BV=64,
|
| 170 |
+
USE_G=g is not None,
|
| 171 |
+
IS_VARLEN=cu_seqlens is not None,
|
| 172 |
+
num_warps=4,
|
| 173 |
+
num_stages=2,
|
| 174 |
+
)
|
| 175 |
+
return o
|
clairic_runtime/gdn/cumsum.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/cumsum.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import triton
|
| 10 |
+
import triton.language as tl
|
| 11 |
+
|
| 12 |
+
from .index import prepare_chunk_indices
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@triton.jit(do_not_specialize=["T"])
|
| 16 |
+
def chunk_local_cumsum_scalar_kernel(
|
| 17 |
+
s: tl.tensor,
|
| 18 |
+
o: tl.tensor,
|
| 19 |
+
scale: tl.tensor,
|
| 20 |
+
cu_seqlens: tl.tensor,
|
| 21 |
+
chunk_indices: tl.tensor,
|
| 22 |
+
T: tl.int32,
|
| 23 |
+
B: tl.constexpr,
|
| 24 |
+
H: tl.constexpr,
|
| 25 |
+
BT: tl.constexpr,
|
| 26 |
+
REVERSE: tl.constexpr,
|
| 27 |
+
HAS_SCALE: tl.constexpr,
|
| 28 |
+
IS_VARLEN: tl.constexpr,
|
| 29 |
+
HEAD_FIRST: tl.constexpr,
|
| 30 |
+
) -> None:
|
| 31 |
+
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
| 32 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 33 |
+
if IS_VARLEN:
|
| 34 |
+
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
|
| 35 |
+
chunk_indices + i_t * 2 + 1
|
| 36 |
+
).to(tl.int32)
|
| 37 |
+
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
|
| 38 |
+
cu_seqlens + i_n + 1
|
| 39 |
+
).to(tl.int32)
|
| 40 |
+
T = eos - bos
|
| 41 |
+
else:
|
| 42 |
+
bos, eos = i_b * T, i_b * T + T
|
| 43 |
+
|
| 44 |
+
if HEAD_FIRST:
|
| 45 |
+
p_s = tl.make_block_ptr(
|
| 46 |
+
s + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,)
|
| 47 |
+
)
|
| 48 |
+
p_o = tl.make_block_ptr(
|
| 49 |
+
o + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,)
|
| 50 |
+
)
|
| 51 |
+
else:
|
| 52 |
+
p_s = tl.make_block_ptr(s + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
| 53 |
+
p_o = tl.make_block_ptr(o + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,))
|
| 54 |
+
# [BT]
|
| 55 |
+
b_s = tl.load(p_s, boundary_check=(0,)).to(tl.float32)
|
| 56 |
+
b_o = tl.cumsum(b_s, axis=0)
|
| 57 |
+
if REVERSE:
|
| 58 |
+
b_z = tl.sum(b_s, axis=0)
|
| 59 |
+
b_o = -b_o + b_z[None] + b_s
|
| 60 |
+
if HAS_SCALE:
|
| 61 |
+
b_o *= scale
|
| 62 |
+
tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0,))
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def chunk_local_cumsum_scalar(
|
| 66 |
+
g: torch.Tensor,
|
| 67 |
+
chunk_size: int,
|
| 68 |
+
reverse: bool = False,
|
| 69 |
+
scale: float = None,
|
| 70 |
+
cu_seqlens: Optional[torch.Tensor] = None,
|
| 71 |
+
head_first: bool = False,
|
| 72 |
+
output_dtype: Optional[torch.dtype] = torch.float,
|
| 73 |
+
chunk_indices: Optional[torch.LongTensor] = None,
|
| 74 |
+
) -> torch.Tensor:
|
| 75 |
+
if head_first:
|
| 76 |
+
B, H, T = g.shape
|
| 77 |
+
else:
|
| 78 |
+
B, T, H = g.shape
|
| 79 |
+
assert chunk_size == 2 ** (
|
| 80 |
+
chunk_size.bit_length() - 1
|
| 81 |
+
), "chunk_size must be a power of 2"
|
| 82 |
+
BT = chunk_size
|
| 83 |
+
if chunk_indices is None and cu_seqlens is not None:
|
| 84 |
+
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
| 85 |
+
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
| 86 |
+
g_org, g = g, torch.empty_like(g, dtype=output_dtype or g.dtype)
|
| 87 |
+
grid = (NT, B * H)
|
| 88 |
+
chunk_local_cumsum_scalar_kernel[grid](
|
| 89 |
+
s=g_org,
|
| 90 |
+
o=g,
|
| 91 |
+
scale=scale,
|
| 92 |
+
cu_seqlens=cu_seqlens,
|
| 93 |
+
chunk_indices=chunk_indices,
|
| 94 |
+
T=T,
|
| 95 |
+
B=B,
|
| 96 |
+
H=H,
|
| 97 |
+
BT=BT,
|
| 98 |
+
HEAD_FIRST=head_first,
|
| 99 |
+
REVERSE=reverse,
|
| 100 |
+
HAS_SCALE=scale is not None,
|
| 101 |
+
IS_VARLEN=cu_seqlens is not None,
|
| 102 |
+
num_warps=8,
|
| 103 |
+
num_stages=3,
|
| 104 |
+
)
|
| 105 |
+
return g
|
| 106 |
+
|
clairic_runtime/gdn/gating.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 2 |
+
# Inference-only adaptation; see LICENSES and NOTICE.
|
| 3 |
+
from typing import Tuple
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
| 11 |
+
# beta_output = b.sigmoid()
|
| 12 |
+
@triton.jit
|
| 13 |
+
def fused_gdn_gating_kernel(
|
| 14 |
+
g: tl.tensor,
|
| 15 |
+
beta_output: tl.tensor,
|
| 16 |
+
A_log: tl.tensor,
|
| 17 |
+
a: tl.tensor,
|
| 18 |
+
b: tl.tensor,
|
| 19 |
+
dt_bias: tl.tensor,
|
| 20 |
+
seq_len: tl.int32,
|
| 21 |
+
stride_a: tl.int32,
|
| 22 |
+
stride_b: tl.int32,
|
| 23 |
+
NUM_HEADS: tl.constexpr,
|
| 24 |
+
beta: tl.constexpr,
|
| 25 |
+
threshold: tl.constexpr,
|
| 26 |
+
BLK_HEADS: tl.constexpr,
|
| 27 |
+
) -> None:
|
| 28 |
+
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
| 29 |
+
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
|
| 30 |
+
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
|
| 31 |
+
mask = head_off < NUM_HEADS
|
| 32 |
+
blk_A_log = tl.load(A_log + head_off, mask=mask)
|
| 33 |
+
blk_a = tl.load(a + i_b * stride_a + head_off, mask=mask)
|
| 34 |
+
blk_b = tl.load(b + i_b * stride_b + head_off, mask=mask)
|
| 35 |
+
blk_bias = tl.load(dt_bias + head_off, mask=mask)
|
| 36 |
+
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
|
| 37 |
+
softplus_x = tl.where(
|
| 38 |
+
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
|
| 39 |
+
)
|
| 40 |
+
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
|
| 41 |
+
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
|
| 42 |
+
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
|
| 43 |
+
tl.store(beta_output + off, blk_beta_output.to(b.dtype.element_ty), mask=mask)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def fused_gdn_gating(
|
| 47 |
+
A_log: torch.Tensor,
|
| 48 |
+
a: torch.Tensor,
|
| 49 |
+
b: torch.Tensor,
|
| 50 |
+
dt_bias: torch.Tensor,
|
| 51 |
+
beta: float = 1.0,
|
| 52 |
+
threshold: float = 20.0,
|
| 53 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 54 |
+
batch, num_heads = a.shape
|
| 55 |
+
seq_len = 1
|
| 56 |
+
stride_a = a.stride(0)
|
| 57 |
+
stride_b = b.stride(0)
|
| 58 |
+
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
|
| 59 |
+
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
|
| 60 |
+
beta_output = torch.empty(1, batch, num_heads, dtype=torch.float32, device=b.device)
|
| 61 |
+
fused_gdn_gating_kernel[grid](
|
| 62 |
+
g,
|
| 63 |
+
beta_output,
|
| 64 |
+
A_log,
|
| 65 |
+
a,
|
| 66 |
+
b,
|
| 67 |
+
dt_bias,
|
| 68 |
+
seq_len,
|
| 69 |
+
stride_a,
|
| 70 |
+
stride_b,
|
| 71 |
+
num_heads,
|
| 72 |
+
beta,
|
| 73 |
+
threshold,
|
| 74 |
+
8,
|
| 75 |
+
num_warps=1,
|
| 76 |
+
)
|
| 77 |
+
return g, beta_output
|
clairic_runtime/gdn/index.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/index.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
import triton
|
| 8 |
+
|
| 9 |
+
from .utils import tensor_cache
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
@tensor_cache
|
| 13 |
+
def prepare_lens(cu_seqlens: torch.LongTensor) -> torch.LongTensor:
|
| 14 |
+
return cu_seqlens[1:] - cu_seqlens[:-1]
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@tensor_cache
|
| 18 |
+
def prepare_chunk_indices(
|
| 19 |
+
cu_seqlens: torch.LongTensor, chunk_size: int
|
| 20 |
+
) -> torch.LongTensor:
|
| 21 |
+
indices = torch.cat(
|
| 22 |
+
[
|
| 23 |
+
torch.arange(n)
|
| 24 |
+
for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist()
|
| 25 |
+
]
|
| 26 |
+
)
|
| 27 |
+
return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@tensor_cache
|
| 31 |
+
def prepare_chunk_offsets(
|
| 32 |
+
cu_seqlens: torch.LongTensor, chunk_size: int
|
| 33 |
+
) -> torch.LongTensor:
|
| 34 |
+
return torch.cat(
|
| 35 |
+
[cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)]
|
| 36 |
+
).cumsum(-1)
|
clairic_runtime/gdn/l2norm.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/modules/l2norm.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import triton
|
| 10 |
+
import triton.language as tl
|
| 11 |
+
|
| 12 |
+
@triton.jit
|
| 13 |
+
def l2norm_fwd_kernel1(
|
| 14 |
+
x: tl.tensor,
|
| 15 |
+
y: tl.tensor,
|
| 16 |
+
D: tl.int32,
|
| 17 |
+
BD: tl.constexpr,
|
| 18 |
+
eps: tl.float32,
|
| 19 |
+
) -> None:
|
| 20 |
+
i_t = tl.program_id(0)
|
| 21 |
+
x += i_t * D
|
| 22 |
+
y += i_t * D
|
| 23 |
+
# Compute mean and variance
|
| 24 |
+
cols = tl.arange(0, BD)
|
| 25 |
+
mask = cols < D
|
| 26 |
+
b_x = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32)
|
| 27 |
+
b_var = tl.sum(b_x * b_x, axis=0)
|
| 28 |
+
b_rstd = 1 / tl.sqrt(b_var + eps)
|
| 29 |
+
# tl.store(Rstd + i_t, rstd)
|
| 30 |
+
# Normalize and apply linear transformation
|
| 31 |
+
b_y = b_x * b_rstd
|
| 32 |
+
tl.store(y + cols, b_y, mask=mask)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@triton.jit(do_not_specialize=["T"])
|
| 36 |
+
def l2norm_fwd_kernel(
|
| 37 |
+
x: tl.tensor,
|
| 38 |
+
y: tl.tensor,
|
| 39 |
+
eps: tl.float32,
|
| 40 |
+
T: tl.int32,
|
| 41 |
+
D: tl.constexpr,
|
| 42 |
+
BT: tl.constexpr,
|
| 43 |
+
BD: tl.constexpr,
|
| 44 |
+
) -> None:
|
| 45 |
+
i_t = tl.program_id(0)
|
| 46 |
+
p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
|
| 47 |
+
b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32)
|
| 48 |
+
b_var = tl.sum(b_x * b_x, axis=1)
|
| 49 |
+
b_y = b_x / tl.sqrt(b_var + eps)[:, None]
|
| 50 |
+
p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0))
|
| 51 |
+
tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1))
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def l2norm_fwd(
|
| 55 |
+
x: torch.Tensor, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None
|
| 56 |
+
) -> torch.Tensor:
|
| 57 |
+
x_shape_og = x.shape
|
| 58 |
+
x = x.view(-1, x.shape[-1])
|
| 59 |
+
# allocate output
|
| 60 |
+
if output_dtype is None:
|
| 61 |
+
y = torch.empty_like(x)
|
| 62 |
+
else:
|
| 63 |
+
y = torch.empty_like(x, dtype=output_dtype)
|
| 64 |
+
assert y.stride(-1) == 1
|
| 65 |
+
T, D = x.shape[0], x.shape[-1]
|
| 66 |
+
# rstd = torch.empty((T,), dtype=torch.float32, device=x.device)
|
| 67 |
+
# Less than 64KB per feature: enqueue fused kernel
|
| 68 |
+
MAX_FUSED_SIZE = 65536 // x.element_size()
|
| 69 |
+
BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D))
|
| 70 |
+
if D > BD:
|
| 71 |
+
raise RuntimeError("This layer doesn't support feature dim >= 64KB.")
|
| 72 |
+
|
| 73 |
+
if D <= 512:
|
| 74 |
+
|
| 75 |
+
def grid(meta: dict[str, int]) -> tuple[int]:
|
| 76 |
+
return (triton.cdiv(T, meta["BT"]),)
|
| 77 |
+
|
| 78 |
+
l2norm_fwd_kernel[grid](
|
| 79 |
+
x,
|
| 80 |
+
y,
|
| 81 |
+
eps,
|
| 82 |
+
T=T,
|
| 83 |
+
D=D,
|
| 84 |
+
BD=BD,
|
| 85 |
+
BT=16,
|
| 86 |
+
num_warps=8,
|
| 87 |
+
num_stages=3,
|
| 88 |
+
)
|
| 89 |
+
else:
|
| 90 |
+
l2norm_fwd_kernel1[(T,)](
|
| 91 |
+
x,
|
| 92 |
+
y,
|
| 93 |
+
eps=eps,
|
| 94 |
+
D=D,
|
| 95 |
+
BD=BD,
|
| 96 |
+
num_warps=8,
|
| 97 |
+
num_stages=3,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
return y.view(x_shape_og)
|
| 101 |
+
|
clairic_runtime/gdn/op.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/op.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
import os
|
| 7 |
+
|
| 8 |
+
import triton
|
| 9 |
+
import triton.language as tl
|
| 10 |
+
import triton.language.extra.libdevice as tldevice
|
| 11 |
+
|
| 12 |
+
if os.environ.get("FLA_USE_FAST_OPS", "0") == "1":
|
| 13 |
+
exp = tldevice.fast_expf
|
| 14 |
+
exp2 = tldevice.exp2
|
| 15 |
+
log = tldevice.fast_logf
|
| 16 |
+
log2 = tldevice.fast_log2f
|
| 17 |
+
else:
|
| 18 |
+
exp = tl.exp
|
| 19 |
+
exp2 = tl.math.exp2
|
| 20 |
+
log = tl.log
|
| 21 |
+
log2 = tl.log2
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
@triton.jit
|
| 26 |
+
def safe_exp(x: tl.tensor) -> tl.tensor:
|
| 27 |
+
return exp(tl.where(x <= 0, x, float("-inf")))
|
| 28 |
+
|
clairic_runtime/gdn/prefill.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any
|
| 2 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 3 |
+
# Inference-only adaptation; see LICENSES and NOTICE.
|
| 4 |
+
from typing import Optional
|
| 5 |
+
import torch
|
| 6 |
+
from .chunk_delta_h import chunk_gated_delta_rule_fwd_h
|
| 7 |
+
from .chunk_fwd import chunk_gated_delta_rule_fwd_intra
|
| 8 |
+
from .chunk_o import chunk_fwd_o
|
| 9 |
+
from .cumsum import chunk_local_cumsum_scalar as chunk_local_cumsum
|
| 10 |
+
CHUNK_SIZE = 64
|
| 11 |
+
SUPPRESS_LEVEL = 0
|
| 12 |
+
|
| 13 |
+
def chunk_gated_delta_rule_fwd(
|
| 14 |
+
q: torch.Tensor,
|
| 15 |
+
k: torch.Tensor,
|
| 16 |
+
v: torch.Tensor,
|
| 17 |
+
g: torch.Tensor,
|
| 18 |
+
beta: torch.Tensor,
|
| 19 |
+
scale: float,
|
| 20 |
+
initial_state: torch.Tensor,
|
| 21 |
+
initial_state_indices: torch.Tensor,
|
| 22 |
+
cu_seqlens: Optional[torch.LongTensor] = None,
|
| 23 |
+
chunk_indices: torch.LongTensor | None = None,
|
| 24 |
+
) -> Any:
|
| 25 |
+
g = chunk_local_cumsum(
|
| 26 |
+
g, chunk_size=CHUNK_SIZE, cu_seqlens=cu_seqlens, chunk_indices=chunk_indices
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
# fused kkt + solve_tril + recompute_w_u
|
| 30 |
+
w, u, A = chunk_gated_delta_rule_fwd_intra(
|
| 31 |
+
k=k,
|
| 32 |
+
v=v,
|
| 33 |
+
g=g,
|
| 34 |
+
beta=beta,
|
| 35 |
+
cu_seqlens=cu_seqlens,
|
| 36 |
+
chunk_indices=chunk_indices,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
h, v_new = chunk_gated_delta_rule_fwd_h(
|
| 40 |
+
k=k,
|
| 41 |
+
w=w,
|
| 42 |
+
u=u,
|
| 43 |
+
g=g,
|
| 44 |
+
initial_state=initial_state,
|
| 45 |
+
initial_state_indices=initial_state_indices,
|
| 46 |
+
cu_seqlens=cu_seqlens,
|
| 47 |
+
chunk_indices=chunk_indices,
|
| 48 |
+
)
|
| 49 |
+
o = chunk_fwd_o(
|
| 50 |
+
q=q,
|
| 51 |
+
k=k,
|
| 52 |
+
v=v_new,
|
| 53 |
+
h=h,
|
| 54 |
+
g=g,
|
| 55 |
+
scale=scale,
|
| 56 |
+
cu_seqlens=cu_seqlens,
|
| 57 |
+
)
|
| 58 |
+
if SUPPRESS_LEVEL < 3:
|
| 59 |
+
return g, o, A, None, h, None
|
| 60 |
+
elif SUPPRESS_LEVEL >= 3:
|
| 61 |
+
return g, o, A, w, h, v_new
|
| 62 |
+
|
clairic_runtime/gdn/recurrent.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 2 |
+
# Inference-only adaptation; see LICENSES and NOTICE.
|
| 3 |
+
import torch
|
| 4 |
+
import triton
|
| 5 |
+
import triton.language as tl
|
| 6 |
+
from .op import exp
|
| 7 |
+
|
| 8 |
+
@triton.jit
|
| 9 |
+
def fused_recurrent_gated_delta_rule_packed_decode_kernel(
|
| 10 |
+
mixed_qkv: tl.tensor,
|
| 11 |
+
a: tl.tensor,
|
| 12 |
+
b: tl.tensor,
|
| 13 |
+
A_log: tl.tensor,
|
| 14 |
+
dt_bias: tl.tensor,
|
| 15 |
+
o: tl.tensor,
|
| 16 |
+
h0: tl.tensor,
|
| 17 |
+
ht: tl.tensor,
|
| 18 |
+
ssm_state_indices: tl.tensor,
|
| 19 |
+
scale: tl.float32,
|
| 20 |
+
stride_mixed_qkv_tok: tl.constexpr,
|
| 21 |
+
stride_a_tok: tl.constexpr,
|
| 22 |
+
stride_b_tok: tl.constexpr,
|
| 23 |
+
stride_init_state_token: tl.constexpr,
|
| 24 |
+
stride_final_state_token: tl.constexpr,
|
| 25 |
+
stride_indices_seq: tl.constexpr,
|
| 26 |
+
H: tl.constexpr,
|
| 27 |
+
HV: tl.constexpr,
|
| 28 |
+
K: tl.constexpr,
|
| 29 |
+
V: tl.constexpr,
|
| 30 |
+
BK: tl.constexpr,
|
| 31 |
+
BV: tl.constexpr,
|
| 32 |
+
SOFTPLUS_THRESHOLD: tl.constexpr,
|
| 33 |
+
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
| 34 |
+
) -> None:
|
| 35 |
+
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
| 36 |
+
i_n, i_hv = i_nh // HV, i_nh % HV
|
| 37 |
+
i_h = i_hv // (HV // H)
|
| 38 |
+
|
| 39 |
+
o_k = tl.arange(0, BK)
|
| 40 |
+
o_v = i_v * BV + tl.arange(0, BV)
|
| 41 |
+
mask_k = o_k < K
|
| 42 |
+
mask_v = o_v < V
|
| 43 |
+
mask_h = mask_v[:, None] & mask_k[None, :]
|
| 44 |
+
|
| 45 |
+
state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64)
|
| 46 |
+
p_o = o + (i_n * HV + i_hv) * V + o_v
|
| 47 |
+
|
| 48 |
+
if state_idx < 0:
|
| 49 |
+
zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty)
|
| 50 |
+
tl.store(p_o, zero, mask=mask_v)
|
| 51 |
+
return
|
| 52 |
+
|
| 53 |
+
p_h0 = h0 + state_idx * stride_init_state_token
|
| 54 |
+
p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
| 55 |
+
b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32)
|
| 56 |
+
|
| 57 |
+
p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok
|
| 58 |
+
q_off = i_h * K + o_k
|
| 59 |
+
k_off = (H * K) + i_h * K + o_k
|
| 60 |
+
v_off = (2 * H * K) + i_hv * V + o_v
|
| 61 |
+
b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32)
|
| 62 |
+
b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32)
|
| 63 |
+
b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32)
|
| 64 |
+
|
| 65 |
+
if USE_QK_L2NORM_IN_KERNEL:
|
| 66 |
+
b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6)
|
| 67 |
+
b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6)
|
| 68 |
+
b_q = b_q * scale
|
| 69 |
+
|
| 70 |
+
a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32)
|
| 71 |
+
b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32)
|
| 72 |
+
A_log_val = tl.load(A_log + i_hv).to(tl.float32)
|
| 73 |
+
dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32)
|
| 74 |
+
x = a_val + dt_bias_val
|
| 75 |
+
softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
|
| 76 |
+
g_val = -tl.exp(A_log_val) * softplus_x
|
| 77 |
+
beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32)
|
| 78 |
+
|
| 79 |
+
b_h *= exp(g_val)
|
| 80 |
+
b_v -= tl.sum(b_h * b_k[None, :], 1)
|
| 81 |
+
b_v *= beta_val
|
| 82 |
+
b_h += b_v[:, None] * b_k[None, :]
|
| 83 |
+
b_o = tl.sum(b_h * b_q[None, :], 1)
|
| 84 |
+
tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v)
|
| 85 |
+
|
| 86 |
+
p_ht = ht + state_idx * stride_final_state_token
|
| 87 |
+
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
| 88 |
+
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def fused_recurrent_gated_delta_rule_packed_decode(
|
| 92 |
+
mixed_qkv: torch.Tensor,
|
| 93 |
+
a: torch.Tensor,
|
| 94 |
+
b: torch.Tensor,
|
| 95 |
+
A_log: torch.Tensor,
|
| 96 |
+
dt_bias: torch.Tensor,
|
| 97 |
+
scale: float,
|
| 98 |
+
initial_state: torch.Tensor,
|
| 99 |
+
out: torch.Tensor,
|
| 100 |
+
ssm_state_indices: torch.Tensor,
|
| 101 |
+
use_qk_l2norm_in_kernel: bool = False,
|
| 102 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 103 |
+
if mixed_qkv.ndim != 2:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})."
|
| 106 |
+
)
|
| 107 |
+
if mixed_qkv.stride(-1) != 1:
|
| 108 |
+
raise ValueError("`mixed_qkv` must be contiguous in the last dim.")
|
| 109 |
+
if a.ndim != 2 or b.ndim != 2:
|
| 110 |
+
raise ValueError(
|
| 111 |
+
f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})."
|
| 112 |
+
)
|
| 113 |
+
if a.stride(-1) != 1 or b.stride(-1) != 1:
|
| 114 |
+
raise ValueError("`a`/`b` must be contiguous in the last dim.")
|
| 115 |
+
if A_log.ndim != 1 or dt_bias.ndim != 1:
|
| 116 |
+
raise ValueError("`A_log`/`dt_bias` must be 1D tensors.")
|
| 117 |
+
if A_log.stride(0) != 1 or dt_bias.stride(0) != 1:
|
| 118 |
+
raise ValueError("`A_log`/`dt_bias` must be contiguous.")
|
| 119 |
+
if ssm_state_indices.ndim != 1:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})."
|
| 122 |
+
)
|
| 123 |
+
if not out.is_contiguous():
|
| 124 |
+
raise ValueError("`out` must be contiguous.")
|
| 125 |
+
|
| 126 |
+
dev = mixed_qkv.device
|
| 127 |
+
if any(
|
| 128 |
+
t.device != dev
|
| 129 |
+
for t in (a, b, A_log, dt_bias, initial_state, out, ssm_state_indices)
|
| 130 |
+
):
|
| 131 |
+
raise ValueError("All inputs must be on the same device.")
|
| 132 |
+
|
| 133 |
+
B = mixed_qkv.shape[0]
|
| 134 |
+
if a.shape[0] != B or b.shape[0] != B:
|
| 135 |
+
raise ValueError(
|
| 136 |
+
"Mismatched batch sizes: "
|
| 137 |
+
f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}."
|
| 138 |
+
)
|
| 139 |
+
if ssm_state_indices.shape[0] != B:
|
| 140 |
+
raise ValueError(
|
| 141 |
+
f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))."
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
if initial_state.ndim != 4:
|
| 145 |
+
raise ValueError(
|
| 146 |
+
f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})."
|
| 147 |
+
)
|
| 148 |
+
if initial_state.stride(-1) != 1:
|
| 149 |
+
raise ValueError("`initial_state` must be contiguous in the last dim.")
|
| 150 |
+
HV, V, K = initial_state.shape[-3:]
|
| 151 |
+
if a.shape[1] != HV or b.shape[1] != HV:
|
| 152 |
+
raise ValueError(
|
| 153 |
+
f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})."
|
| 154 |
+
)
|
| 155 |
+
if A_log.numel() != HV or dt_bias.numel() != HV:
|
| 156 |
+
raise ValueError(
|
| 157 |
+
f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})."
|
| 158 |
+
)
|
| 159 |
+
if out.shape != (B, 1, HV, V):
|
| 160 |
+
raise ValueError(
|
| 161 |
+
f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})."
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
qkv_dim = mixed_qkv.shape[1]
|
| 165 |
+
qk_dim = qkv_dim - HV * V
|
| 166 |
+
if qk_dim <= 0 or qk_dim % 2 != 0:
|
| 167 |
+
raise ValueError(
|
| 168 |
+
f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}."
|
| 169 |
+
)
|
| 170 |
+
q_dim = qk_dim // 2
|
| 171 |
+
if q_dim % K != 0:
|
| 172 |
+
raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.")
|
| 173 |
+
H = q_dim // K
|
| 174 |
+
if H <= 0 or HV % H != 0:
|
| 175 |
+
raise ValueError(
|
| 176 |
+
f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}."
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
BK = triton.next_power_of_2(K)
|
| 180 |
+
if triton.cdiv(K, BK) != 1:
|
| 181 |
+
raise ValueError(
|
| 182 |
+
f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})."
|
| 183 |
+
)
|
| 184 |
+
BV = min(triton.next_power_of_2(V), 32)
|
| 185 |
+
num_stages = 3
|
| 186 |
+
num_warps = 1
|
| 187 |
+
|
| 188 |
+
stride_mixed_qkv_tok = mixed_qkv.stride(0)
|
| 189 |
+
stride_a_tok = a.stride(0)
|
| 190 |
+
stride_b_tok = b.stride(0)
|
| 191 |
+
stride_init_state_token = initial_state.stride(0)
|
| 192 |
+
stride_final_state_token = initial_state.stride(0)
|
| 193 |
+
stride_indices_seq = ssm_state_indices.stride(0)
|
| 194 |
+
|
| 195 |
+
NV = triton.cdiv(V, BV)
|
| 196 |
+
grid = (NV, B * HV)
|
| 197 |
+
fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
|
| 198 |
+
mixed_qkv=mixed_qkv,
|
| 199 |
+
a=a,
|
| 200 |
+
b=b,
|
| 201 |
+
A_log=A_log,
|
| 202 |
+
dt_bias=dt_bias,
|
| 203 |
+
o=out,
|
| 204 |
+
h0=initial_state,
|
| 205 |
+
ht=initial_state,
|
| 206 |
+
ssm_state_indices=ssm_state_indices,
|
| 207 |
+
scale=scale,
|
| 208 |
+
stride_mixed_qkv_tok=stride_mixed_qkv_tok,
|
| 209 |
+
stride_a_tok=stride_a_tok,
|
| 210 |
+
stride_b_tok=stride_b_tok,
|
| 211 |
+
stride_init_state_token=stride_init_state_token,
|
| 212 |
+
stride_final_state_token=stride_final_state_token,
|
| 213 |
+
stride_indices_seq=stride_indices_seq,
|
| 214 |
+
H=H,
|
| 215 |
+
HV=HV,
|
| 216 |
+
K=K,
|
| 217 |
+
V=V,
|
| 218 |
+
BK=BK,
|
| 219 |
+
BV=BV,
|
| 220 |
+
SOFTPLUS_THRESHOLD=20.0,
|
| 221 |
+
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
| 222 |
+
num_warps=num_warps,
|
| 223 |
+
num_stages=num_stages,
|
| 224 |
+
)
|
| 225 |
+
return out, initial_state
|
| 226 |
+
|
clairic_runtime/gdn/utils.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 2 |
+
# Standalone helper plumbing; see LICENSES and NOTICE for retained notices.
|
| 3 |
+
|
| 4 |
+
"""Device, cache, and launch helpers for the standalone inference kernels."""
|
| 5 |
+
|
| 6 |
+
import contextlib
|
| 7 |
+
import functools
|
| 8 |
+
import inspect
|
| 9 |
+
import os
|
| 10 |
+
from collections.abc import Callable
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import triton
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
FLA_CACHE_RESULTS = os.getenv("FLA_CACHE_RESULTS", "1") == "1"
|
| 18 |
+
SUPPORTS_AUTOTUNE_CACHE = "cache_results" in inspect.signature(triton.autotune).parameters
|
| 19 |
+
autotune_cache_kwargs = (
|
| 20 |
+
{"cache_results": FLA_CACHE_RESULTS} if SUPPORTS_AUTOTUNE_CACHE else {}
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@functools.lru_cache(maxsize=1)
|
| 25 |
+
def _backend() -> str:
|
| 26 |
+
try:
|
| 27 |
+
return triton.runtime.driver.active.get_current_target().backend
|
| 28 |
+
except RuntimeError:
|
| 29 |
+
return "cpu"
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
is_nvidia = _backend() == "cuda"
|
| 33 |
+
is_nvidia_hopper = is_nvidia and (
|
| 34 |
+
"NVIDIA H" in torch.cuda.get_device_name(0)
|
| 35 |
+
or torch.cuda.get_device_capability()[0] >= 9
|
| 36 |
+
)
|
| 37 |
+
is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@functools.lru_cache(maxsize=None)
|
| 41 |
+
def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:
|
| 42 |
+
required = {
|
| 43 |
+
"ADA": 101376,
|
| 44 |
+
"AMPERE": 166912,
|
| 45 |
+
"HOPPER": 232448,
|
| 46 |
+
}.get(arch.upper(), 102400)
|
| 47 |
+
try:
|
| 48 |
+
properties = triton.runtime.driver.active.utils.get_device_properties(tensor_idx)
|
| 49 |
+
return properties["max_shared_mem"] >= required
|
| 50 |
+
except Exception:
|
| 51 |
+
return False
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def tensor_cache(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 55 |
+
"""Retain the source's four-entry cache keyed by input object identity."""
|
| 56 |
+
cache_entries: list[tuple[tuple[Any, ...], dict[str, Any], Any]] = []
|
| 57 |
+
cache_size = 4
|
| 58 |
+
|
| 59 |
+
@functools.wraps(fn)
|
| 60 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 61 |
+
nonlocal cache_entries, cache_size
|
| 62 |
+
for i, entry in enumerate(cache_entries):
|
| 63 |
+
last_args, last_kwargs, last_result = entry
|
| 64 |
+
if len(args) == len(last_args) and len(kwargs) == len(last_kwargs):
|
| 65 |
+
if all(a is b for a, b in zip(args, last_args)) and all(
|
| 66 |
+
k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items()
|
| 67 |
+
):
|
| 68 |
+
cache_entries = (
|
| 69 |
+
cache_entries[:i]
|
| 70 |
+
+ cache_entries[i + 1 :]
|
| 71 |
+
+ [(args, kwargs, last_result)]
|
| 72 |
+
)
|
| 73 |
+
return last_result
|
| 74 |
+
result = fn(*args, **kwargs)
|
| 75 |
+
if len(cache_entries) >= cache_size:
|
| 76 |
+
cache_entries = cache_entries[1:]
|
| 77 |
+
cache_entries.append((args, kwargs, result))
|
| 78 |
+
return result
|
| 79 |
+
|
| 80 |
+
return wrapper
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def input_guard(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 84 |
+
"""Make tensor inputs contiguous and retain their CUDA device context."""
|
| 85 |
+
|
| 86 |
+
@functools.wraps(fn)
|
| 87 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 88 |
+
contiguous_args = (
|
| 89 |
+
value.contiguous() if isinstance(value, torch.Tensor) else value
|
| 90 |
+
for value in args
|
| 91 |
+
)
|
| 92 |
+
contiguous_kwargs = {
|
| 93 |
+
key: value.contiguous() if isinstance(value, torch.Tensor) else value
|
| 94 |
+
for key, value in kwargs.items()
|
| 95 |
+
}
|
| 96 |
+
tensor = next((value for value in args if isinstance(value, torch.Tensor)), None)
|
| 97 |
+
if tensor is None:
|
| 98 |
+
tensor = next(
|
| 99 |
+
(value for value in kwargs.values() if isinstance(value, torch.Tensor)),
|
| 100 |
+
None,
|
| 101 |
+
)
|
| 102 |
+
context = (
|
| 103 |
+
torch.cuda.device(tensor.device.index)
|
| 104 |
+
if tensor is not None and tensor.is_cuda
|
| 105 |
+
else contextlib.nullcontext()
|
| 106 |
+
)
|
| 107 |
+
with context:
|
| 108 |
+
return fn(*contiguous_args, **contiguous_kwargs)
|
| 109 |
+
|
| 110 |
+
return wrapper
|
clairic_runtime/gdn/wy_fast.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/wy_fast.py
|
| 2 |
+
# -*- coding: utf-8 -*-
|
| 3 |
+
# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang
|
| 4 |
+
# Standalone import layout; see LICENSES and NOTICE for retained notices.
|
| 5 |
+
|
| 6 |
+
from typing import Optional, Tuple
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import triton
|
| 10 |
+
import triton.language as tl
|
| 11 |
+
|
| 12 |
+
from .index import prepare_chunk_indices
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
# @triton.autotune(
|
| 16 |
+
# configs=[
|
| 17 |
+
# triton.Config({}, num_warps=num_warps, num_stages=num_stages)
|
| 18 |
+
# for num_warps in [2, 4, 8]
|
| 19 |
+
# for num_stages in [2, 3, 4]
|
| 20 |
+
# ],
|
| 21 |
+
# key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"],
|
| 22 |
+
# )
|
| 23 |
+
@triton.jit(do_not_specialize=["T"])
|
| 24 |
+
def recompute_w_u_fwd_kernel(
|
| 25 |
+
k: tl.tensor,
|
| 26 |
+
v: tl.tensor,
|
| 27 |
+
beta: tl.tensor,
|
| 28 |
+
w: tl.tensor,
|
| 29 |
+
u: tl.tensor,
|
| 30 |
+
A: tl.tensor,
|
| 31 |
+
g: tl.tensor,
|
| 32 |
+
cu_seqlens: tl.tensor,
|
| 33 |
+
chunk_indices: tl.tensor,
|
| 34 |
+
T: tl.int32,
|
| 35 |
+
H: tl.constexpr,
|
| 36 |
+
Hg: tl.constexpr,
|
| 37 |
+
K: tl.constexpr,
|
| 38 |
+
V: tl.constexpr,
|
| 39 |
+
BT: tl.constexpr,
|
| 40 |
+
BK: tl.constexpr,
|
| 41 |
+
BV: tl.constexpr,
|
| 42 |
+
IS_VARLEN: tl.constexpr,
|
| 43 |
+
) -> None:
|
| 44 |
+
i_t, i_bh = tl.program_id(0), tl.program_id(1)
|
| 45 |
+
i_b, i_h = i_bh // H, i_bh % H
|
| 46 |
+
if IS_VARLEN:
|
| 47 |
+
i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load(
|
| 48 |
+
chunk_indices + i_t * 2 + 1
|
| 49 |
+
).to(tl.int32)
|
| 50 |
+
bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load(
|
| 51 |
+
cu_seqlens + i_n + 1
|
| 52 |
+
).to(tl.int32)
|
| 53 |
+
T = eos - bos
|
| 54 |
+
else:
|
| 55 |
+
bos, eos = i_b * T, i_b * T + T
|
| 56 |
+
p_beta = tl.make_block_ptr(
|
| 57 |
+
beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)
|
| 58 |
+
)
|
| 59 |
+
p_g = tl.make_block_ptr(g + (bos * H + i_h), (T,), (H,), (i_t * BT,), (BT,), (0,))
|
| 60 |
+
p_A = tl.make_block_ptr(
|
| 61 |
+
A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0)
|
| 62 |
+
)
|
| 63 |
+
b_beta = tl.load(p_beta, boundary_check=(0,))
|
| 64 |
+
b_A = tl.load(p_A, boundary_check=(0, 1))
|
| 65 |
+
b_g = tl.exp(tl.load(p_g, boundary_check=(0,)))
|
| 66 |
+
|
| 67 |
+
for i_v in range(tl.cdiv(V, BV)):
|
| 68 |
+
p_v = tl.make_block_ptr(
|
| 69 |
+
v + (bos * H + i_h) * V,
|
| 70 |
+
(T, V),
|
| 71 |
+
(H * V, 1),
|
| 72 |
+
(i_t * BT, i_v * BV),
|
| 73 |
+
(BT, BV),
|
| 74 |
+
(1, 0),
|
| 75 |
+
)
|
| 76 |
+
p_u = tl.make_block_ptr(
|
| 77 |
+
u + (bos * H + i_h) * V,
|
| 78 |
+
(T, V),
|
| 79 |
+
(H * V, 1),
|
| 80 |
+
(i_t * BT, i_v * BV),
|
| 81 |
+
(BT, BV),
|
| 82 |
+
(1, 0),
|
| 83 |
+
)
|
| 84 |
+
b_v = tl.load(p_v, boundary_check=(0, 1))
|
| 85 |
+
b_vb = (b_v * b_beta[:, None]).to(b_v.dtype)
|
| 86 |
+
b_u = tl.dot(b_A, b_vb, allow_tf32=False)
|
| 87 |
+
tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1))
|
| 88 |
+
|
| 89 |
+
for i_k in range(tl.cdiv(K, BK)):
|
| 90 |
+
p_k = tl.make_block_ptr(
|
| 91 |
+
k + (bos * Hg + i_h // (H // Hg)) * K,
|
| 92 |
+
(T, K),
|
| 93 |
+
(Hg * K, 1),
|
| 94 |
+
(i_t * BT, i_k * BK),
|
| 95 |
+
(BT, BK),
|
| 96 |
+
(1, 0),
|
| 97 |
+
)
|
| 98 |
+
p_w = tl.make_block_ptr(
|
| 99 |
+
w + (bos * H + i_h) * K,
|
| 100 |
+
(T, K),
|
| 101 |
+
(H * K, 1),
|
| 102 |
+
(i_t * BT, i_k * BK),
|
| 103 |
+
(BT, BK),
|
| 104 |
+
(1, 0),
|
| 105 |
+
)
|
| 106 |
+
b_k = tl.load(p_k, boundary_check=(0, 1))
|
| 107 |
+
b_kb = (b_k * b_beta[:, None] * b_g[:, None]).to(b_k.dtype)
|
| 108 |
+
b_w = tl.dot(b_A, b_kb)
|
| 109 |
+
tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1))
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def recompute_w_u_fwd(
|
| 113 |
+
k: torch.Tensor,
|
| 114 |
+
v: torch.Tensor,
|
| 115 |
+
beta: torch.Tensor,
|
| 116 |
+
g_cumsum: torch.Tensor,
|
| 117 |
+
A: torch.Tensor,
|
| 118 |
+
cu_seqlens: Optional[torch.LongTensor],
|
| 119 |
+
chunk_indices: torch.LongTensor | None = None,
|
| 120 |
+
) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 121 |
+
B, T, Hg, K, V = *k.shape, v.shape[-1]
|
| 122 |
+
H = v.shape[-2]
|
| 123 |
+
BT = A.shape[-1]
|
| 124 |
+
|
| 125 |
+
if chunk_indices is None and cu_seqlens is not None:
|
| 126 |
+
chunk_indices = prepare_chunk_indices(cu_seqlens, BT)
|
| 127 |
+
NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices)
|
| 128 |
+
BK = 64
|
| 129 |
+
BV = 64
|
| 130 |
+
u = torch.empty_like(v)
|
| 131 |
+
w = k.new_empty(B, T, H, K)
|
| 132 |
+
recompute_w_u_fwd_kernel[(NT, B * H)](
|
| 133 |
+
k=k,
|
| 134 |
+
v=v,
|
| 135 |
+
beta=beta,
|
| 136 |
+
w=w,
|
| 137 |
+
u=u,
|
| 138 |
+
A=A,
|
| 139 |
+
g=g_cumsum,
|
| 140 |
+
cu_seqlens=cu_seqlens,
|
| 141 |
+
chunk_indices=chunk_indices,
|
| 142 |
+
T=T,
|
| 143 |
+
H=H,
|
| 144 |
+
Hg=Hg,
|
| 145 |
+
K=K,
|
| 146 |
+
V=V,
|
| 147 |
+
BT=BT,
|
| 148 |
+
BK=BK,
|
| 149 |
+
BV=BV,
|
| 150 |
+
IS_VARLEN=cu_seqlens is not None,
|
| 151 |
+
num_warps=4,
|
| 152 |
+
num_stages=3,
|
| 153 |
+
)
|
| 154 |
+
return w, u
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
fwd_recompute_w_u = recompute_w_u_fwd
|
clairic_runtime/gdn_pool.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Engine-owned native linear-cache storage; dense caches remain fresh native."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from typing import Any
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class NativeStatePool:
|
| 10 |
+
"""One maxB allocation per linear layer, attached to fresh DynamicCache objects.
|
| 11 |
+
|
| 12 |
+
The caller serializes requests on its model worker and must not reuse a cache
|
| 13 |
+
after its request ends. The initialized flags prevent native prefill from
|
| 14 |
+
allocating duplicate state; has_previous_state=False retains original prefill.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def __init__(self, backbone: Any, max_batch: int) -> None:
|
| 18 |
+
if max_batch < 1:
|
| 19 |
+
raise ValueError("Positive maximum batch required")
|
| 20 |
+
self.max_batch, self.closed = max_batch, False
|
| 21 |
+
self.language = backbone.model.language_model
|
| 22 |
+
self.states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
|
| 23 |
+
for index, layer in enumerate(self.language.layers):
|
| 24 |
+
if layer.layer_type != "linear_attention":
|
| 25 |
+
continue
|
| 26 |
+
gdn = layer.linear_attn
|
| 27 |
+
weight = gdn.in_proj_qkv.weight
|
| 28 |
+
conv = torch.zeros(max_batch, gdn.conv_dim, gdn.conv_kernel_size,
|
| 29 |
+
dtype=weight.dtype, device=weight.device)
|
| 30 |
+
source = torch.zeros(max_batch, gdn.num_v_heads, gdn.head_v_dim, gdn.head_k_dim,
|
| 31 |
+
dtype=torch.float32, device=weight.device)
|
| 32 |
+
self.states[index] = (conv, source.transpose(-1, -2))
|
| 33 |
+
if not self.states:
|
| 34 |
+
raise ValueError("Expected native linear-attention layers")
|
| 35 |
+
|
| 36 |
+
def attach(self, cache: Any, batch: int, *, previous: bool = False) -> Any:
|
| 37 |
+
if self.closed or not 1 <= batch <= self.max_batch:
|
| 38 |
+
raise RuntimeError("Closed pool or batch outside owned state")
|
| 39 |
+
if len(cache.layers) != len(self.language.layers):
|
| 40 |
+
raise ValueError("Cache layer count differs from language model")
|
| 41 |
+
for index, (conv, recurrent) in self.states.items():
|
| 42 |
+
layer = cache.layers[index]
|
| 43 |
+
# Attach to empty native linear layers only; replacing live storage
|
| 44 |
+
# would silently discard state or retain an unaccounted allocation.
|
| 45 |
+
if layer.is_conv_states_initialized or layer.is_recurrent_states_initialized:
|
| 46 |
+
raise ValueError("Fresh uninitialized native cache required")
|
| 47 |
+
layer.conv_states = conv[:batch]
|
| 48 |
+
layer.recurrent_states = recurrent[:batch]
|
| 49 |
+
layer.is_conv_states_initialized = True
|
| 50 |
+
layer.is_recurrent_states_initialized = True
|
| 51 |
+
layer.has_previous_state = previous
|
| 52 |
+
layer.dtype, layer.device = conv.dtype, conv.device
|
| 53 |
+
layer.max_batch_size, layer.conv_kernel_size = batch, conv.shape[-1]
|
| 54 |
+
return cache
|
| 55 |
+
|
| 56 |
+
def new_cache(self, batch: int, *, previous: bool = False) -> Any:
|
| 57 |
+
if self.closed:
|
| 58 |
+
raise RuntimeError("Closed pool")
|
| 59 |
+
from transformers.cache_utils import DynamicCache
|
| 60 |
+
|
| 61 |
+
return self.attach(DynamicCache(config=self.language.config), batch, previous=previous)
|
| 62 |
+
|
| 63 |
+
def owns(self, cache: Any, batch: int) -> bool:
|
| 64 |
+
if self.closed or not 1 <= batch <= self.max_batch or cache is None:
|
| 65 |
+
return False
|
| 66 |
+
if len(cache.layers) != len(self.language.layers):
|
| 67 |
+
return False
|
| 68 |
+
for index, (conv, recurrent) in self.states.items():
|
| 69 |
+
layer = cache.layers[index]
|
| 70 |
+
for actual, owned in ((layer.conv_states, conv), (layer.recurrent_states, recurrent)):
|
| 71 |
+
if (actual is None or actual.data_ptr() != owned.data_ptr()
|
| 72 |
+
or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
|
| 73 |
+
or actual.dtype != owned.dtype or actual.device != owned.device
|
| 74 |
+
or actual.storage_offset() != owned.storage_offset()
|
| 75 |
+
or tuple(actual.shape) != (batch, *owned.shape[1:])
|
| 76 |
+
or tuple(actual.stride()) != tuple(owned.stride())):
|
| 77 |
+
return False
|
| 78 |
+
if not (layer.is_conv_states_initialized and layer.is_recurrent_states_initialized):
|
| 79 |
+
return False
|
| 80 |
+
return True
|
| 81 |
+
|
| 82 |
+
@property
|
| 83 |
+
def storage_bytes(self) -> int:
|
| 84 |
+
return sum(value.untyped_storage().nbytes() for states in self.states.values() for value in states)
|
| 85 |
+
|
| 86 |
+
def zero(self) -> None:
|
| 87 |
+
if self.closed:
|
| 88 |
+
raise RuntimeError("Closed pool")
|
| 89 |
+
for conv, recurrent in self.states.values():
|
| 90 |
+
conv.zero_()
|
| 91 |
+
recurrent.zero_()
|
| 92 |
+
|
| 93 |
+
def close(self) -> None:
|
| 94 |
+
self.states.clear()
|
| 95 |
+
self.language = None
|
| 96 |
+
self.closed = True
|
clairic_runtime/kernels.py
CHANGED
|
@@ -172,32 +172,179 @@ def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor,
|
|
| 172 |
gate.view(batch, length, q_heads * dim))
|
| 173 |
|
| 174 |
|
| 175 |
-
@triton.jit
|
| 176 |
-
def
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
tl.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
|
| 193 |
|
| 194 |
def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
|
| 195 |
eps: float) -> torch.Tensor:
|
| 196 |
output = torch.empty_like(x)
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
)
|
| 202 |
return output
|
| 203 |
|
|
@@ -476,3 +623,60 @@ def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor |
|
|
| 476 |
return c
|
| 477 |
else:
|
| 478 |
raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
gate.view(batch, length, q_heads * dim))
|
| 173 |
|
| 174 |
|
| 175 |
+
@triton.jit
|
| 176 |
+
def _layer_norm_fwd_1pass_kernel(
|
| 177 |
+
X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor,
|
| 178 |
+
Mean: tl.tensor, Rstd: tl.tensor,
|
| 179 |
+
stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64,
|
| 180 |
+
M: tl.int32, N: tl.constexpr, eps: tl.float32,
|
| 181 |
+
BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr,
|
| 182 |
+
HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr,
|
| 183 |
+
IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr,
|
| 184 |
+
USE_GDC: tl.constexpr = False,
|
| 185 |
+
) -> None:
|
| 186 |
+
if USE_GDC:
|
| 187 |
+
tl.extra.cuda.gdc_wait()
|
| 188 |
+
row_start = tl.program_id(0) * ROWS_PER_BLOCK
|
| 189 |
+
group = tl.program_id(1)
|
| 190 |
+
rows = row_start + tl.arange(0, ROWS_PER_BLOCK)
|
| 191 |
+
cols = tl.arange(0, BLOCK_N)
|
| 192 |
+
row_offsets = rows[:, None] * stride_x_row
|
| 193 |
+
col_offsets = cols[None, :] + group * N
|
| 194 |
+
X_base = X + row_offsets + col_offsets
|
| 195 |
+
Y_base = Y + rows[:, None] * stride_y_row + col_offsets
|
| 196 |
+
row_mask = rows[:, None] < M
|
| 197 |
+
col_mask = cols[None, :] < N
|
| 198 |
+
mask = row_mask & col_mask
|
| 199 |
+
x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32)
|
| 200 |
+
if HAS_Z and (not NORM_BEFORE_GATE):
|
| 201 |
+
Z_base = Z + rows[:, None] * stride_z_row + col_offsets
|
| 202 |
+
z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
|
| 203 |
+
if ACTIVATION == 'swish' or ACTIVATION == 'silu':
|
| 204 |
+
x *= z * tl.sigmoid(z)
|
| 205 |
+
elif ACTIVATION == 'sigmoid':
|
| 206 |
+
x *= tl.sigmoid(z)
|
| 207 |
+
if not IS_RMS_NORM:
|
| 208 |
+
mean = tl.sum(x, axis=1) / N
|
| 209 |
+
mean_offsets = group * M + rows
|
| 210 |
+
mean_mask = rows < M
|
| 211 |
+
tl.store(Mean + mean_offsets, mean, mask=mean_mask)
|
| 212 |
+
xbar = tl.where(mask, x - mean[:, None], 0.0)
|
| 213 |
+
var = tl.sum(xbar * xbar, axis=1) / N
|
| 214 |
+
else:
|
| 215 |
+
xbar = tl.where(mask, x, 0.0)
|
| 216 |
+
var = tl.sum(xbar * xbar, axis=1) / N
|
| 217 |
+
mean = 0.0
|
| 218 |
+
rstd = tl.rsqrt(var + eps)
|
| 219 |
+
rstd_offsets = group * M + rows
|
| 220 |
+
rstd_mask = rows < M
|
| 221 |
+
tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask)
|
| 222 |
+
w_offsets = cols + group * N
|
| 223 |
+
w_mask = cols < N
|
| 224 |
+
w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
|
| 225 |
+
if HAS_BIAS:
|
| 226 |
+
b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
|
| 227 |
+
if not IS_RMS_NORM:
|
| 228 |
+
x_hat = (x - mean[:, None]) * rstd[:, None]
|
| 229 |
+
else:
|
| 230 |
+
x_hat = x * rstd[:, None]
|
| 231 |
+
y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :]
|
| 232 |
+
if HAS_Z and NORM_BEFORE_GATE:
|
| 233 |
+
Z_base = Z + rows[:, None] * stride_z_row + col_offsets
|
| 234 |
+
z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
|
| 235 |
+
if ACTIVATION == 'swish' or ACTIVATION == 'silu':
|
| 236 |
+
y *= z * tl.sigmoid(z)
|
| 237 |
+
elif ACTIVATION == 'sigmoid':
|
| 238 |
+
y *= tl.sigmoid(z)
|
| 239 |
+
tl.store(Y_base, y, mask=mask)
|
| 240 |
+
if USE_GDC:
|
| 241 |
+
tl.extra.cuda.gdc_launch_dependents()
|
| 242 |
|
| 243 |
|
| 244 |
def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
|
| 245 |
eps: float) -> torch.Tensor:
|
| 246 |
output = torch.empty_like(x)
|
| 247 |
+
rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device)
|
| 248 |
+
block_n = triton.next_power_of_2(x.shape[1])
|
| 249 |
+
pdl = torch.cuda.get_device_capability(x.device)[0] >= 9
|
| 250 |
+
launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {}
|
| 251 |
+
_layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)](
|
| 252 |
+
x, output, weight, None, z, None, rstd,
|
| 253 |
+
x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps,
|
| 254 |
+
BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True,
|
| 255 |
+
NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish",
|
| 256 |
+
num_warps=min(max(block_n // 256, 1), 8), **launch,
|
| 257 |
+
)
|
| 258 |
+
return output
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@triton.jit
|
| 262 |
+
def _fused_sigmoid_mul_kernel(
|
| 263 |
+
output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor,
|
| 264 |
+
gate_stride_row: tl.int64, gate_stride_head: tl.int64,
|
| 265 |
+
hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr,
|
| 266 |
+
) -> None:
|
| 267 |
+
"""Fuse sigmoid(gate) * attn_output into a single kernel."""
|
| 268 |
+
pid_row = tl.program_id(0).to(tl.int64)
|
| 269 |
+
pid_block = tl.program_id(1)
|
| 270 |
+
offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H)
|
| 271 |
+
mask = offsets < hidden_dim
|
| 272 |
+
head = offsets // HEAD_DIM
|
| 273 |
+
d = offsets - head * HEAD_DIM
|
| 274 |
+
attn_off = pid_row * hidden_dim + offsets
|
| 275 |
+
attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32)
|
| 276 |
+
gate_off = pid_row * gate_stride_row + head * gate_stride_head + d
|
| 277 |
+
g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32)
|
| 278 |
+
result = attn * tl.sigmoid(g)
|
| 279 |
+
tl.store(output_ptr + attn_off, result, mask=mask)
|
| 280 |
+
|
| 281 |
+
def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor:
|
| 282 |
+
"""
|
| 283 |
+
Fused sigmoid-mul for attention output gating.
|
| 284 |
+
|
| 285 |
+
Equivalent to: attn_output * sigmoid(gate)
|
| 286 |
+
|
| 287 |
+
The production Qwen3.5 path passes a 3D strided gate. A single hidden-block
|
| 288 |
+
Triton kernel handles both that path and flat contiguous inputs.
|
| 289 |
+
|
| 290 |
+
When inplace=True, writes result back to attn_output and returns it.
|
| 291 |
+
|
| 292 |
+
Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim)
|
| 293 |
+
and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate
|
| 294 |
+
via explicit strides without requiring a contiguous copy.
|
| 295 |
+
"""
|
| 296 |
+
if gate.ndim == 3 and attn_output.ndim == 2:
|
| 297 |
+
num_tokens, num_heads, head_dim = gate.shape
|
| 298 |
+
hidden_dim = num_heads * head_dim
|
| 299 |
+
assert attn_output.shape == (num_tokens, hidden_dim)
|
| 300 |
+
gate_stride_row = gate.stride(0)
|
| 301 |
+
gate_stride_head = gate.stride(1)
|
| 302 |
+
else:
|
| 303 |
+
assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape'
|
| 304 |
+
hidden_dim = attn_output.shape[-1]
|
| 305 |
+
num_tokens = attn_output.numel() // hidden_dim
|
| 306 |
+
head_dim = hidden_dim
|
| 307 |
+
gate_stride_row = hidden_dim
|
| 308 |
+
gate_stride_head = hidden_dim
|
| 309 |
+
out = attn_output if inplace else torch.empty_like(attn_output)
|
| 310 |
+
block_h = 1024 if num_tokens < 1024 else 2048
|
| 311 |
+
grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
|
| 312 |
+
_fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4)
|
| 313 |
+
return out
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
@triton.jit
|
| 317 |
+
def _silu_mul_kernel(
|
| 318 |
+
gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor,
|
| 319 |
+
elements: tl.constexpr, BLOCK: tl.constexpr,
|
| 320 |
+
) -> None:
|
| 321 |
+
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
| 322 |
+
mask = offsets < elements
|
| 323 |
+
gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
| 324 |
+
up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
| 325 |
+
# Preserve the qualified CUDA --use_fast_math operations, including FTZ.
|
| 326 |
+
output = tl.inline_asm_elementwise(
|
| 327 |
+
"{ .reg .f32 exponent, denominator, activated; "
|
| 328 |
+
"mul.ftz.f32 exponent, $1, 0fBFB8AA3B; "
|
| 329 |
+
"ex2.approx.ftz.f32 exponent, exponent; "
|
| 330 |
+
"add.ftz.f32 denominator, exponent, 0f3F800000; "
|
| 331 |
+
"div.approx.ftz.f32 activated, $1, denominator; "
|
| 332 |
+
"mul.ftz.f32 $0, $2, activated; }",
|
| 333 |
+
constraints="=f,f,f", args=[gate, up], dtype=tl.float32,
|
| 334 |
+
is_pure=True, pack=1,
|
| 335 |
+
)
|
| 336 |
+
tl.store(output_ptr + offsets, output, mask=mask)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
|
| 340 |
+
"""Match the qualified fused CUDA SiLU-mul before the BF16 output store."""
|
| 341 |
+
if (gate.shape != up.shape or gate.device != up.device
|
| 342 |
+
or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16
|
| 343 |
+
or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()):
|
| 344 |
+
raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors")
|
| 345 |
+
output = torch.empty_like(gate)
|
| 346 |
+
_silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)](
|
| 347 |
+
gate, up, output, gate.numel(), BLOCK=2048, num_warps=8,
|
| 348 |
)
|
| 349 |
return output
|
| 350 |
|
|
|
|
| 623 |
return c
|
| 624 |
else:
|
| 625 |
raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')
|
| 626 |
+
|
| 627 |
+
|
| 628 |
+
|
| 629 |
+
@triton.jit(do_not_specialize=["count"])
|
| 630 |
+
def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor,
|
| 631 |
+
count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None:
|
| 632 |
+
offsets = tl.program_id(0) * block + tl.arange(0, block)
|
| 633 |
+
current = tl.load(value + offsets, offsets < count, other=0)
|
| 634 |
+
clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL),
|
| 635 |
+
limit, propagate_nan=tl.PropagateNan.ALL)
|
| 636 |
+
upper = clipped.to(tl.bfloat16)
|
| 637 |
+
lower = (current - upper.to(tl.float32)).to(tl.bfloat16)
|
| 638 |
+
lower = tl.where(current == 0, upper, lower)
|
| 639 |
+
tl.store(high + offsets, upper, offsets < count)
|
| 640 |
+
tl.store(low + offsets, lower, offsets < count)
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
|
| 644 |
+
def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 645 |
+
if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous():
|
| 646 |
+
raise ValueError("Expected contiguous CUDA FP32 residuals")
|
| 647 |
+
high = torch.empty_like(value, dtype=torch.bfloat16)
|
| 648 |
+
low = torch.empty_like(value, dtype=torch.bfloat16)
|
| 649 |
+
count = value.numel()
|
| 650 |
+
if count:
|
| 651 |
+
_bf16_pair_kernel[(triton.cdiv(count, 1024),)](
|
| 652 |
+
value, high, low, count, torch.finfo(torch.bfloat16).max, 1024,
|
| 653 |
+
num_warps=4, enable_fp_fusion=False,
|
| 654 |
+
)
|
| 655 |
+
return high, low
|
| 656 |
+
|
| 657 |
+
|
| 658 |
+
|
| 659 |
+
@triton.jit(do_not_specialize=["count"])
|
| 660 |
+
def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor,
|
| 661 |
+
count: tl.int32, block: tl.constexpr) -> None:
|
| 662 |
+
offsets = tl.program_id(0) * block + tl.arange(0, block)
|
| 663 |
+
upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32)
|
| 664 |
+
lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32)
|
| 665 |
+
tl.store(output + offsets, upper + lower, offsets < count)
|
| 666 |
+
|
| 667 |
+
|
| 668 |
+
|
| 669 |
+
def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor:
|
| 670 |
+
if (not first.is_cuda or second.device != first.device or second.shape != first.shape
|
| 671 |
+
or first.dtype not in (torch.bfloat16, torch.float32)
|
| 672 |
+
or second.dtype != torch.bfloat16
|
| 673 |
+
or not first.is_contiguous() or not second.is_contiguous()):
|
| 674 |
+
raise ValueError("Expected aligned contiguous CUDA residuals")
|
| 675 |
+
output = torch.empty_like(first, dtype=torch.float32)
|
| 676 |
+
count = first.numel()
|
| 677 |
+
if count:
|
| 678 |
+
_residual_kernel[(triton.cdiv(count, 1024),)](
|
| 679 |
+
first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False,
|
| 680 |
+
)
|
| 681 |
+
return output
|
| 682 |
+
|
clairic_runtime/math.py
CHANGED
|
@@ -29,10 +29,22 @@ from transformers.models.qwen3_5.modeling_qwen3_5 import (
|
|
| 29 |
eager_attention_forward, torch_causal_conv1d_update,
|
| 30 |
)
|
| 31 |
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
| 34 |
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
|
| 35 |
raise ValueError("Expected aligned matrices with equal dtype")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
from .kernels import _matmul_persistent_triton
|
| 37 |
return _matmul_persistent_triton(a, b, bias)
|
| 38 |
|
|
@@ -83,15 +95,40 @@ class ReadoutMath(TorchDispatchMode):
|
|
| 83 |
return function(*args, **kwargs)
|
| 84 |
|
| 85 |
|
| 86 |
-
def _linear_forward(original: Callable[..., Any], lm_head: bool
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
|
| 88 |
if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
|
| 89 |
or input.dtype != self.weight.dtype):
|
|
|
|
| 90 |
return original(input)
|
| 91 |
shape = input.shape[:-1]
|
| 92 |
flat = input.reshape(-1, input.shape[-1])
|
| 93 |
if lm_head:
|
| 94 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
else:
|
| 96 |
output = _matmul(flat, self.weight.T, self.bias)
|
| 97 |
return output.reshape(*shape, self.weight.shape[0])
|
|
@@ -99,11 +136,29 @@ def _linear_forward(original: Callable[..., Any], lm_head: bool) -> Callable[...
|
|
| 99 |
|
| 100 |
|
| 101 |
def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
|
|
|
|
|
|
|
|
|
|
| 102 |
value = x.float()
|
| 103 |
value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
|
| 104 |
return (value * (1.0 + self.weight.float())).to(self.weight.dtype)
|
| 105 |
|
| 106 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
|
| 108 |
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 109 |
attention_mask: torch.Tensor | None = None,
|
|
@@ -119,13 +174,17 @@ def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
|
|
| 119 |
states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
|
| 120 |
position_ids=position_ids, past_key_values=past_key_values,
|
| 121 |
position_embeddings=position_embeddings, **kwargs)
|
| 122 |
-
merged =
|
| 123 |
states = self.post_attention_layernorm(merged)
|
| 124 |
-
return
|
| 125 |
|
| 126 |
|
| 127 |
def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
|
| 128 |
gate, up = self.gate_proj(x), self.up_proj(x)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))
|
| 130 |
|
| 131 |
|
|
@@ -146,6 +205,7 @@ def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor,
|
|
| 146 |
def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
|
| 147 |
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 148 |
attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
|
|
|
|
| 149 |
**kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
|
| 150 |
input_shape = hidden_states.shape[:-1]
|
| 151 |
hidden_shape = (*input_shape, -1, self.head_dim)
|
|
@@ -165,11 +225,20 @@ def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
|
|
| 165 |
key, value = past_key_values.update(key, value, self.layer_idx)
|
| 166 |
interface = ALL_ATTENTION_FUNCTIONS.get_interface(
|
| 167 |
self.config._attn_implementation, eager_attention_forward)
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
output = output.reshape(*input_shape, -1).contiguous()
|
| 172 |
-
output
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
return self.o_proj(output), weights
|
| 174 |
|
| 175 |
|
|
@@ -201,27 +270,16 @@ def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor,
|
|
| 201 |
mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
|
| 202 |
if cached:
|
| 203 |
mixed = mixed[:, :, -length:]
|
| 204 |
-
|
| 205 |
-
(self.key_dim, self.key_dim, self.value_dim), -1)
|
| 206 |
-
query = query.reshape(batch, length, -1, self.head_k_dim)
|
| 207 |
-
key = key.reshape(batch, length, -1, self.head_k_dim)
|
| 208 |
-
value = value.reshape(batch, length, -1, self.head_v_dim)
|
| 209 |
-
beta = b.float().sigmoid() if cached and length == 1 else b.sigmoid()
|
| 210 |
-
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
| 211 |
-
if self.num_v_heads // self.num_k_heads > 1:
|
| 212 |
-
groups = self.num_v_heads // self.num_k_heads
|
| 213 |
-
query, key = query.repeat_interleave(groups, 2), key.repeat_interleave(groups, 2)
|
| 214 |
if cached and length == 1:
|
| 215 |
-
output, state =
|
| 216 |
-
|
| 217 |
-
use_qk_l2norm_in_kernel=True)
|
| 218 |
else:
|
| 219 |
-
output, state =
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
cu_seqlens=kwargs.get("cu_seq_lens_q"))
|
| 223 |
if cache_params is not None:
|
| 224 |
-
|
| 225 |
output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
|
| 226 |
return self.out_proj(output.reshape(batch, length, -1))
|
| 227 |
|
|
@@ -309,6 +367,8 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
|
|
| 309 |
forwards: list[tuple[nn.Module, bool, Any, Any]] = []
|
| 310 |
convolutions: list[tuple[Any, Any, Any]] = []
|
| 311 |
caches: dict[torch.device, torch.Tensor] = {}
|
|
|
|
|
|
|
| 312 |
closed = False
|
| 313 |
|
| 314 |
def restore() -> None:
|
|
@@ -316,6 +376,7 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
|
|
| 316 |
if closed:
|
| 317 |
return
|
| 318 |
closed = True
|
|
|
|
| 319 |
for module, original_prefill, original_update in reversed(convolutions):
|
| 320 |
module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
|
| 321 |
for module, had_override, original, installed in reversed(forwards):
|
|
@@ -325,6 +386,9 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
|
|
| 325 |
else:
|
| 326 |
del module.forward
|
| 327 |
caches.clear()
|
|
|
|
|
|
|
|
|
|
| 328 |
|
| 329 |
def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
|
| 330 |
parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
|
|
@@ -364,13 +428,24 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
|
|
| 364 |
elif isinstance(module, Qwen3_5MLP):
|
| 365 |
bind(module, _mlp, "x", module.gate_proj.weight)
|
| 366 |
elif isinstance(module, Qwen3_5Attention):
|
| 367 |
-
|
|
|
|
|
|
|
| 368 |
elif isinstance(module, Qwen3_5GatedDeltaNet):
|
| 369 |
-
bind(module, _gdn, parameter=module.in_proj_qkv.weight)
|
| 370 |
bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
|
| 371 |
convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
|
| 372 |
module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
|
| 373 |
bind(rotary, text_rotary, "x")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
lm_head = backbone.get_output_embeddings()
|
| 375 |
roots = (backbone,) if head is None else (backbone, head)
|
| 376 |
seen: set[int] = set()
|
|
@@ -382,7 +457,7 @@ def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None =
|
|
| 382 |
if module.weight.dtype not in (torch.bfloat16, torch.float32):
|
| 383 |
continue
|
| 384 |
original = module.forward
|
| 385 |
-
installed = MethodType(_linear_forward(original, module is lm_head), module)
|
| 386 |
forwards.append((module, "forward" in module.__dict__, original, installed))
|
| 387 |
module.forward = installed
|
| 388 |
except BaseException:
|
|
|
|
| 29 |
eager_attention_forward, torch_causal_conv1d_update,
|
| 30 |
)
|
| 31 |
|
| 32 |
+
from .attention_metadata import DecodeMetadata
|
| 33 |
+
from .recurrent_metadata import scope as recurrent_metadata_scope
|
| 34 |
+
|
| 35 |
|
| 36 |
def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
|
| 37 |
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
|
| 38 |
raise ValueError("Expected aligned matrices with equal dtype")
|
| 39 |
+
if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
|
| 40 |
+
and a.is_contiguous() and b.transpose(0, 1).is_contiguous()
|
| 41 |
+
and b.shape[1] >= 16):
|
| 42 |
+
from deep_gemm import bf16_gemm_nn
|
| 43 |
+
output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype)
|
| 44 |
+
bf16_gemm_nn(a, b, output)
|
| 45 |
+
if bias is not None:
|
| 46 |
+
output += bias
|
| 47 |
+
return output
|
| 48 |
from .kernels import _matmul_persistent_triton
|
| 49 |
return _matmul_persistent_triton(a, b, bias)
|
| 50 |
|
|
|
|
| 95 |
return function(*args, **kwargs)
|
| 96 |
|
| 97 |
|
| 98 |
+
def _linear_forward(original: Callable[..., Any], lm_head: bool,
|
| 99 |
+
transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]:
|
| 100 |
+
cache: dict[str, Any] = {}
|
| 101 |
+
if lm_head and transpose_caches is not None:
|
| 102 |
+
transpose_caches.append(cache)
|
| 103 |
def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
|
| 104 |
if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
|
| 105 |
or input.dtype != self.weight.dtype):
|
| 106 |
+
cache.clear()
|
| 107 |
return original(input)
|
| 108 |
shape = input.shape[:-1]
|
| 109 |
flat = input.reshape(-1, input.shape[-1])
|
| 110 |
if lm_head:
|
| 111 |
+
weight = self.weight
|
| 112 |
+
version = None
|
| 113 |
+
if not torch.is_grad_enabled() and not torch.compiler.is_compiling():
|
| 114 |
+
try:
|
| 115 |
+
version = weight._version
|
| 116 |
+
except RuntimeError:
|
| 117 |
+
pass
|
| 118 |
+
if version is None:
|
| 119 |
+
cache.clear()
|
| 120 |
+
transposed = weight.T.contiguous()
|
| 121 |
+
else:
|
| 122 |
+
stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None
|
| 123 |
+
key = (id(weight), version, weight.data_ptr(), weight.storage_offset(),
|
| 124 |
+
weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream)
|
| 125 |
+
if cache.get("weight") is not weight or cache.get("key") != key:
|
| 126 |
+
cache.clear()
|
| 127 |
+
cache["transposed"] = weight.T.contiguous()
|
| 128 |
+
cache["key"] = key
|
| 129 |
+
cache["weight"] = weight
|
| 130 |
+
transposed = cache["transposed"]
|
| 131 |
+
output = _matmul(flat.contiguous(), transposed).float()
|
| 132 |
else:
|
| 133 |
output = _matmul(flat, self.weight.T, self.bias)
|
| 134 |
return output.reshape(*shape, self.weight.shape[0])
|
|
|
|
| 136 |
|
| 137 |
|
| 138 |
def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
|
| 139 |
+
if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32):
|
| 140 |
+
from .normalization import rms_norm
|
| 141 |
+
return rms_norm(x, self.weight, self.eps)
|
| 142 |
value = x.float()
|
| 143 |
value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
|
| 144 |
return (value * (1.0 + self.weight.float())).to(self.weight.dtype)
|
| 145 |
|
| 146 |
|
| 147 |
+
def _residual_sum(first: torch.Tensor, second: torch.Tensor,
|
| 148 |
+
dtype: torch.dtype) -> torch.Tensor:
|
| 149 |
+
if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda
|
| 150 |
+
and first.device == second.device and first.shape == second.shape
|
| 151 |
+
and first.dtype in (torch.bfloat16, torch.float32)
|
| 152 |
+
and second.dtype == torch.bfloat16
|
| 153 |
+
and first.is_contiguous() and second.is_contiguous()
|
| 154 |
+
and first.numel() <= 2**31 - 1
|
| 155 |
+
and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
|
| 156 |
+
from .kernels import residual_sum
|
| 157 |
+
return residual_sum(first, second)
|
| 158 |
+
return first.to(dtype).float() + second.float()
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
|
| 162 |
def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
|
| 163 |
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 164 |
attention_mask: torch.Tensor | None = None,
|
|
|
|
| 174 |
states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
|
| 175 |
position_ids=position_ids, past_key_values=past_key_values,
|
| 176 |
position_embeddings=position_embeddings, **kwargs)
|
| 177 |
+
merged = _residual_sum(residual, states, dtype)
|
| 178 |
states = self.post_attention_layernorm(merged)
|
| 179 |
+
return _residual_sum(merged, self.mlp(states), dtype)
|
| 180 |
|
| 181 |
|
| 182 |
def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
|
| 183 |
gate, up = self.gate_proj(x), self.up_proj(x)
|
| 184 |
+
if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16
|
| 185 |
+
and gate.is_contiguous() and up.is_contiguous()):
|
| 186 |
+
from .kernels import silu_mul
|
| 187 |
+
return self.down_proj(silu_mul(gate, up))
|
| 188 |
return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))
|
| 189 |
|
| 190 |
|
|
|
|
| 205 |
def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
|
| 206 |
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
| 207 |
attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
|
| 208 |
+
*, metadata: DecodeMetadata | None = None,
|
| 209 |
**kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
|
| 210 |
input_shape = hidden_states.shape[:-1]
|
| 211 |
hidden_shape = (*input_shape, -1, self.head_dim)
|
|
|
|
| 225 |
key, value = past_key_values.update(key, value, self.layer_idx)
|
| 226 |
interface = ALL_ATTENTION_FUNCTIONS.get_interface(
|
| 227 |
self.config._attn_implementation, eager_attention_forward)
|
| 228 |
+
if metadata is not None:
|
| 229 |
+
output, weights = metadata.attend(interface, self, query, key, value, attention_mask,
|
| 230 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 231 |
+
scaling=self.scaling, **kwargs)
|
| 232 |
+
else:
|
| 233 |
+
output, weights = interface(self, query, key, value, attention_mask,
|
| 234 |
+
dropout=0.0 if not self.training else self.attention_dropout,
|
| 235 |
+
scaling=self.scaling, **kwargs)
|
| 236 |
output = output.reshape(*input_shape, -1).contiguous()
|
| 237 |
+
if output.is_cuda and output.dtype == torch.bfloat16:
|
| 238 |
+
from .kernels import fused_sigmoid_mul
|
| 239 |
+
output = fused_sigmoid_mul(output, gate, inplace=True)
|
| 240 |
+
else:
|
| 241 |
+
output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype)
|
| 242 |
return self.o_proj(output), weights
|
| 243 |
|
| 244 |
|
|
|
|
| 270 |
mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
|
| 271 |
if cached:
|
| 272 |
mixed = mixed[:, :, -length:]
|
| 273 |
+
from .gdn.adapter import decode, prefill, store_state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
if cached and length == 1:
|
| 275 |
+
output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
|
| 276 |
+
recurrent_state)
|
|
|
|
| 277 |
else:
|
| 278 |
+
output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
|
| 279 |
+
self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim,
|
| 280 |
+
recurrent_state if cached else None, attention_mask)
|
|
|
|
| 281 |
if cache_params is not None:
|
| 282 |
+
store_state(cache_params, self.layer_idx, state)
|
| 283 |
output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
|
| 284 |
return self.out_proj(output.reshape(batch, length, -1))
|
| 285 |
|
|
|
|
| 367 |
forwards: list[tuple[nn.Module, bool, Any, Any]] = []
|
| 368 |
convolutions: list[tuple[Any, Any, Any]] = []
|
| 369 |
caches: dict[torch.device, torch.Tensor] = {}
|
| 370 |
+
transpose_caches: list[dict[str, Any]] = []
|
| 371 |
+
attention_metadata = DecodeMetadata()
|
| 372 |
closed = False
|
| 373 |
|
| 374 |
def restore() -> None:
|
|
|
|
| 376 |
if closed:
|
| 377 |
return
|
| 378 |
closed = True
|
| 379 |
+
attention_metadata.close()
|
| 380 |
for module, original_prefill, original_update in reversed(convolutions):
|
| 381 |
module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
|
| 382 |
for module, had_override, original, installed in reversed(forwards):
|
|
|
|
| 386 |
else:
|
| 387 |
del module.forward
|
| 388 |
caches.clear()
|
| 389 |
+
for cache in transpose_caches:
|
| 390 |
+
cache.clear()
|
| 391 |
+
transpose_caches.clear()
|
| 392 |
|
| 393 |
def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
|
| 394 |
parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
|
|
|
|
| 428 |
elif isinstance(module, Qwen3_5MLP):
|
| 429 |
bind(module, _mlp, "x", module.gate_proj.weight)
|
| 430 |
elif isinstance(module, Qwen3_5Attention):
|
| 431 |
+
def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any:
|
| 432 |
+
return _dense(self, *args, metadata=attention_metadata, **kwargs)
|
| 433 |
+
bind(module, dense, parameter=module.q_proj.weight)
|
| 434 |
elif isinstance(module, Qwen3_5GatedDeltaNet):
|
| 435 |
+
bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True)
|
| 436 |
bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
|
| 437 |
convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
|
| 438 |
module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
|
| 439 |
bind(rotary, text_rotary, "x")
|
| 440 |
+
original_language_forward = language.forward
|
| 441 |
+
|
| 442 |
+
def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
|
| 443 |
+
with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self):
|
| 444 |
+
return original_language_forward(*args, **kwargs)
|
| 445 |
+
|
| 446 |
+
installed_language = MethodType(language_forward, language)
|
| 447 |
+
forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language))
|
| 448 |
+
language.forward = installed_language
|
| 449 |
lm_head = backbone.get_output_embeddings()
|
| 450 |
roots = (backbone,) if head is None else (backbone, head)
|
| 451 |
seen: set[int] = set()
|
|
|
|
| 457 |
if module.weight.dtype not in (torch.bfloat16, torch.float32):
|
| 458 |
continue
|
| 459 |
original = module.forward
|
| 460 |
+
installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module)
|
| 461 |
forwards.append((module, "forward" in module.__dict__, original, installed))
|
| 462 |
module.forward = installed
|
| 463 |
except BaseException:
|
clairic_runtime/native.py
CHANGED
|
@@ -7,8 +7,8 @@ import concurrent.futures
|
|
| 7 |
import copy
|
| 8 |
import json
|
| 9 |
import threading
|
| 10 |
-
from collections.abc import Callable
|
| 11 |
-
from contextlib import nullcontext
|
| 12 |
from collections import defaultdict
|
| 13 |
from pathlib import Path
|
| 14 |
from typing import Any
|
|
@@ -18,10 +18,52 @@ from . import model as cm
|
|
| 18 |
from .readout import load_head, read_record
|
| 19 |
from .math import ReadoutMath, install as install_math
|
| 20 |
from .vision_cache import VisionBatchCache
|
|
|
|
| 21 |
|
| 22 |
_FLASH_ATTENTION = 'kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0'
|
| 23 |
|
| 24 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
class _ConstrainedScores:
|
| 26 |
"""Capture probabilities after applying the schema's grammar constraints."""
|
| 27 |
|
|
@@ -30,12 +72,34 @@ class _ConstrainedScores:
|
|
| 30 |
self.score_ids = score_ids
|
| 31 |
self.chosen: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
|
| 32 |
self.first: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor:
|
|
|
|
|
|
|
| 35 |
masked = self.processor(input_ids, scores.clone())
|
| 36 |
logprobs = torch.log_softmax(masked.float(), dim=-1)
|
| 37 |
-
if not self.
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
self.first = [
|
| 40 |
[
|
| 41 |
(float(value), token, None)
|
|
@@ -43,13 +107,9 @@ class _ConstrainedScores:
|
|
| 43 |
]
|
| 44 |
for row in selected
|
| 45 |
]
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
self.chosen, choices.cpu().tolist(), values, strict=True
|
| 50 |
-
):
|
| 51 |
-
row.append((float(value), int(token), None))
|
| 52 |
-
return masked
|
| 53 |
|
| 54 |
|
| 55 |
class _CancellationCriteria:
|
|
@@ -76,20 +136,23 @@ class TransformersEngine:
|
|
| 76 |
head: cm.ClairicHead,
|
| 77 |
tokenizer: Any,
|
| 78 |
*,
|
| 79 |
-
batch_size: int =
|
| 80 |
text_batch_size: int | None = None,
|
| 81 |
-
batch_token_budget: int =
|
| 82 |
) -> None:
|
| 83 |
import xgrammar as xgr
|
| 84 |
|
| 85 |
self.backbone = backbone.eval()
|
| 86 |
self.head = head.to(backbone.device).float().eval()
|
| 87 |
self.tokenizer = tokenizer
|
| 88 |
-
|
|
|
|
| 89 |
self.text_batch_size = (
|
| 90 |
self.batch_size if text_batch_size is None else max(1, int(text_batch_size))
|
| 91 |
)
|
| 92 |
-
self.batch_token_budget = max(1, int(
|
|
|
|
|
|
|
| 93 |
self.device, self.dtype = backbone.device, backbone.dtype
|
| 94 |
self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
|
| 95 |
self.image_token = int(backbone.config.image_token_id)
|
|
@@ -106,6 +169,8 @@ class TransformersEngine:
|
|
| 106 |
self._uncached_image_features = self.backbone.model.get_image_features
|
| 107 |
self.backbone.model.get_image_features = self._image_features
|
| 108 |
self._restore_math: Callable[[], None] | None = None
|
|
|
|
|
|
|
| 109 |
|
| 110 |
def begin_request(self) -> None:
|
| 111 |
self._vision_cache.begin()
|
|
@@ -137,9 +202,9 @@ class TransformersEngine:
|
|
| 137 |
*,
|
| 138 |
device: str = "cuda",
|
| 139 |
dtype: Any = None,
|
| 140 |
-
batch_size: int =
|
| 141 |
text_batch_size: int | None = None,
|
| 142 |
-
batch_token_budget: int =
|
| 143 |
**kwargs: Any,
|
| 144 |
) -> TransformersEngine:
|
| 145 |
from transformers import Qwen3_5ForConditionalGeneration
|
|
@@ -152,23 +217,31 @@ class TransformersEngine:
|
|
| 152 |
if isinstance(dtype, str):
|
| 153 |
dtype = getattr(torch, dtype)
|
| 154 |
kwargs.setdefault("device_map", {"": device})
|
|
|
|
| 155 |
if "attn_implementation" not in kwargs:
|
| 156 |
attention = "sdpa"
|
| 157 |
if str(device).startswith("cuda") and dtype == torch.bfloat16:
|
| 158 |
if torch.cuda.get_device_capability(torch.device(device))[0] == 9:
|
| 159 |
attention = _FLASH_ATTENTION
|
|
|
|
| 160 |
kwargs["attn_implementation"] = {
|
| 161 |
"text_config": attention, "vision_config": attention,
|
| 162 |
}
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
|
|
|
| 169 |
engine = cls(
|
| 170 |
backbone,
|
| 171 |
-
load_head(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
tokenizer,
|
| 173 |
batch_size=batch_size,
|
| 174 |
text_batch_size=text_batch_size,
|
|
@@ -176,6 +249,9 @@ class TransformersEngine:
|
|
| 176 |
)
|
| 177 |
try:
|
| 178 |
engine._restore_math = install_math(backbone, engine.head)
|
|
|
|
|
|
|
|
|
|
| 179 |
except BaseException:
|
| 180 |
engine.shutdown()
|
| 181 |
raise
|
|
@@ -307,16 +383,21 @@ class TransformersEngine:
|
|
| 307 |
config.max_new_tokens, config.use_cache = maximum, True
|
| 308 |
config.eos_token_id = list(stop_ids)
|
| 309 |
config.pad_token_id = self.tokenizer.pad_token_id or self.eos
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
if _cancelled is not None and _cancelled.is_set():
|
| 319 |
raise RuntimeError("Inference cancelled")
|
|
|
|
| 320 |
continuations = (
|
| 321 |
generated[:, inputs["input_ids"].shape[1] :].cpu().tolist()
|
| 322 |
)
|
|
@@ -429,9 +510,27 @@ class TransformersEngine:
|
|
| 429 |
lambda cancelled: self._read_typed(item, media, _cancelled=cancelled)
|
| 430 |
)
|
| 431 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 432 |
def shutdown(self) -> None:
|
| 433 |
try:
|
| 434 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 435 |
self._vision_cache.clear()
|
| 436 |
if self.backbone.model.get_image_features == self._image_features:
|
| 437 |
self.backbone.model.get_image_features = self._uncached_image_features
|
|
|
|
| 7 |
import copy
|
| 8 |
import json
|
| 9 |
import threading
|
| 10 |
+
from collections.abc import Callable, Iterator
|
| 11 |
+
from contextlib import contextmanager, nullcontext
|
| 12 |
from collections import defaultdict
|
| 13 |
from pathlib import Path
|
| 14 |
from typing import Any
|
|
|
|
| 18 |
from .readout import load_head, read_record
|
| 19 |
from .math import ReadoutMath, install as install_math
|
| 20 |
from .vision_cache import VisionBatchCache
|
| 21 |
+
from .decoder_graph import prepare as prepare_decoder_graphs
|
| 22 |
|
| 23 |
_FLASH_ATTENTION = 'kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0'
|
| 24 |
|
| 25 |
|
| 26 |
+
def _batch_defaults(backbone: Any) -> tuple[int, int]:
|
| 27 |
+
config = getattr(backbone.config, "text_config", None)
|
| 28 |
+
if (backbone.device.type == "cuda" and backbone.dtype == torch.bfloat16
|
| 29 |
+
and config is not None and config._attn_implementation == _FLASH_ATTENTION):
|
| 30 |
+
if config.hidden_size == 4096:
|
| 31 |
+
return 64, 65536
|
| 32 |
+
if config.hidden_size == 5120:
|
| 33 |
+
return 32, 32768
|
| 34 |
+
return 8, 8192
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
_ATTENTION_KERNEL_LOCK = threading.RLock()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@contextmanager
|
| 41 |
+
def _default_attention_kernel() -> Iterator[None]:
|
| 42 |
+
"""Select the pinned public attention build while initializing its loader."""
|
| 43 |
+
import os
|
| 44 |
+
from huggingface_hub import snapshot_download
|
| 45 |
+
from transformers.modeling_flash_attention_utils import lazy_import_flash_attention
|
| 46 |
+
|
| 47 |
+
repository, revision = _FLASH_ATTENTION.rsplit("@", 1)
|
| 48 |
+
build_name = "torch-stable-abi210-cu130-x86_64-linux"
|
| 49 |
+
snapshot = snapshot_download(repository, revision=revision,
|
| 50 |
+
allow_patterns=[f"build/{build_name}/**"])
|
| 51 |
+
build = Path(snapshot) / "build" / build_name
|
| 52 |
+
with _ATTENTION_KERNEL_LOCK:
|
| 53 |
+
previous = os.environ.get("LOCAL_KERNELS")
|
| 54 |
+
entries = previous.split(":") if previous else []
|
| 55 |
+
entries = [entry for entry in entries if entry.partition("=")[0] != repository]
|
| 56 |
+
os.environ["LOCAL_KERNELS"] = ":".join([*entries, f"{repository}={build}"])
|
| 57 |
+
try:
|
| 58 |
+
yield
|
| 59 |
+
lazy_import_flash_attention(implementation=_FLASH_ATTENTION)
|
| 60 |
+
finally:
|
| 61 |
+
if previous is None:
|
| 62 |
+
os.environ.pop("LOCAL_KERNELS", None)
|
| 63 |
+
else:
|
| 64 |
+
os.environ["LOCAL_KERNELS"] = previous
|
| 65 |
+
|
| 66 |
+
|
| 67 |
class _ConstrainedScores:
|
| 68 |
"""Capture probabilities after applying the schema's grammar constraints."""
|
| 69 |
|
|
|
|
| 72 |
self.score_ids = score_ids
|
| 73 |
self.chosen: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
|
| 74 |
self.first: list[list[tuple[float, int, None]]] = [[] for _ in range(size)]
|
| 75 |
+
self._tokens: list[torch.Tensor] = []
|
| 76 |
+
self._logprobs: list[torch.Tensor] = []
|
| 77 |
+
self._first_values: torch.Tensor | None = None
|
| 78 |
|
| 79 |
def __call__(self, input_ids: torch.Tensor, scores: torch.Tensor) -> torch.Tensor:
|
| 80 |
+
if self.processor.prefilled and input_ids.is_cuda:
|
| 81 |
+
input_ids = input_ids[:, -1:].cpu()
|
| 82 |
masked = self.processor(input_ids, scores.clone())
|
| 83 |
logprobs = torch.log_softmax(masked.float(), dim=-1)
|
| 84 |
+
if not self._tokens:
|
| 85 |
+
self._first_values = logprobs[:, self.score_ids]
|
| 86 |
+
choices = masked.argmax(dim=-1)
|
| 87 |
+
self._tokens.append(choices)
|
| 88 |
+
self._logprobs.append(logprobs.gather(1, choices[:, None])[:, 0])
|
| 89 |
+
return masked
|
| 90 |
+
|
| 91 |
+
def finalize(self) -> None:
|
| 92 |
+
if not self._tokens:
|
| 93 |
+
return
|
| 94 |
+
tokens = torch.stack(self._tokens, dim=1).cpu().tolist()
|
| 95 |
+
values = torch.stack(self._logprobs, dim=1).cpu().tolist()
|
| 96 |
+
self.chosen = [
|
| 97 |
+
[(float(value), int(token), None)
|
| 98 |
+
for token, value in zip(row_tokens, row_values, strict=True)]
|
| 99 |
+
for row_tokens, row_values in zip(tokens, values, strict=True)
|
| 100 |
+
]
|
| 101 |
+
if self._first_values is not None:
|
| 102 |
+
selected = self._first_values.cpu().tolist()
|
| 103 |
self.first = [
|
| 104 |
[
|
| 105 |
(float(value), token, None)
|
|
|
|
| 107 |
]
|
| 108 |
for row in selected
|
| 109 |
]
|
| 110 |
+
self._tokens.clear()
|
| 111 |
+
self._logprobs.clear()
|
| 112 |
+
self._first_values = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
|
| 114 |
|
| 115 |
class _CancellationCriteria:
|
|
|
|
| 136 |
head: cm.ClairicHead,
|
| 137 |
tokenizer: Any,
|
| 138 |
*,
|
| 139 |
+
batch_size: int | None = None,
|
| 140 |
text_batch_size: int | None = None,
|
| 141 |
+
batch_token_budget: int | None = None,
|
| 142 |
) -> None:
|
| 143 |
import xgrammar as xgr
|
| 144 |
|
| 145 |
self.backbone = backbone.eval()
|
| 146 |
self.head = head.to(backbone.device).float().eval()
|
| 147 |
self.tokenizer = tokenizer
|
| 148 |
+
default_size, default_budget = _batch_defaults(backbone)
|
| 149 |
+
self.batch_size = max(1, int(default_size if batch_size is None else batch_size))
|
| 150 |
self.text_batch_size = (
|
| 151 |
self.batch_size if text_batch_size is None else max(1, int(text_batch_size))
|
| 152 |
)
|
| 153 |
+
self.batch_token_budget = max(1, int(
|
| 154 |
+
default_budget if batch_token_budget is None else batch_token_budget
|
| 155 |
+
))
|
| 156 |
self.device, self.dtype = backbone.device, backbone.dtype
|
| 157 |
self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>")
|
| 158 |
self.image_token = int(backbone.config.image_token_id)
|
|
|
|
| 169 |
self._uncached_image_features = self.backbone.model.get_image_features
|
| 170 |
self.backbone.model.get_image_features = self._image_features
|
| 171 |
self._restore_math: Callable[[], None] | None = None
|
| 172 |
+
self._decoder_graphs: Any = None
|
| 173 |
+
self._decoder_cleanup: Any = None
|
| 174 |
|
| 175 |
def begin_request(self) -> None:
|
| 176 |
self._vision_cache.begin()
|
|
|
|
| 202 |
*,
|
| 203 |
device: str = "cuda",
|
| 204 |
dtype: Any = None,
|
| 205 |
+
batch_size: int | None = None,
|
| 206 |
text_batch_size: int | None = None,
|
| 207 |
+
batch_token_budget: int | None = None,
|
| 208 |
**kwargs: Any,
|
| 209 |
) -> TransformersEngine:
|
| 210 |
from transformers import Qwen3_5ForConditionalGeneration
|
|
|
|
| 217 |
if isinstance(dtype, str):
|
| 218 |
dtype = getattr(torch, dtype)
|
| 219 |
kwargs.setdefault("device_map", {"": device})
|
| 220 |
+
default_flash = False
|
| 221 |
if "attn_implementation" not in kwargs:
|
| 222 |
attention = "sdpa"
|
| 223 |
if str(device).startswith("cuda") and dtype == torch.bfloat16:
|
| 224 |
if torch.cuda.get_device_capability(torch.device(device))[0] == 9:
|
| 225 |
attention = _FLASH_ATTENTION
|
| 226 |
+
default_flash = True
|
| 227 |
kwargs["attn_implementation"] = {
|
| 228 |
"text_config": attention, "vision_config": attention,
|
| 229 |
}
|
| 230 |
+
with _default_attention_kernel() if default_flash else nullcontext():
|
| 231 |
+
backbone = _NativeQwen3_5ForConditionalGeneration.from_pretrained(
|
| 232 |
+
path,
|
| 233 |
+
dtype=dtype,
|
| 234 |
+
local_files_only=True,
|
| 235 |
+
**kwargs,
|
| 236 |
+
)
|
| 237 |
engine = cls(
|
| 238 |
backbone,
|
| 239 |
+
load_head(
|
| 240 |
+
path,
|
| 241 |
+
match_bfloat16=(
|
| 242 |
+
backbone.device.type == "cuda" and backbone.dtype == torch.bfloat16
|
| 243 |
+
),
|
| 244 |
+
),
|
| 245 |
tokenizer,
|
| 246 |
batch_size=batch_size,
|
| 247 |
text_batch_size=text_batch_size,
|
|
|
|
| 249 |
)
|
| 250 |
try:
|
| 251 |
engine._restore_math = install_math(backbone, engine.head)
|
| 252 |
+
engine._decoder_graphs = engine._worker.submit(
|
| 253 |
+
prepare_decoder_graphs, backbone, max(engine.batch_size, engine.text_batch_size)
|
| 254 |
+
).result()
|
| 255 |
except BaseException:
|
| 256 |
engine.shutdown()
|
| 257 |
raise
|
|
|
|
| 383 |
config.max_new_tokens, config.use_cache = maximum, True
|
| 384 |
config.eos_token_id = list(stop_ids)
|
| 385 |
config.pad_token_id = self.tokenizer.pad_token_id or self.eos
|
| 386 |
+
scope = (self._decoder_graphs.request(len(chunk))
|
| 387 |
+
if self._decoder_graphs is not None else nullcontext(None))
|
| 388 |
+
with scope as native_cache:
|
| 389 |
+
generated = self.backbone.generate(
|
| 390 |
+
**inputs,
|
| 391 |
+
**({"past_key_values": native_cache} if native_cache is not None else {}),
|
| 392 |
+
generation_config=config,
|
| 393 |
+
logits_processor=[scores],
|
| 394 |
+
stopping_criteria=[_CancellationCriteria(_cancelled)]
|
| 395 |
+
if _cancelled is not None
|
| 396 |
+
else [],
|
| 397 |
+
)
|
| 398 |
if _cancelled is not None and _cancelled.is_set():
|
| 399 |
raise RuntimeError("Inference cancelled")
|
| 400 |
+
scores.finalize()
|
| 401 |
continuations = (
|
| 402 |
generated[:, inputs["input_ids"].shape[1] :].cpu().tolist()
|
| 403 |
)
|
|
|
|
| 510 |
lambda cancelled: self._read_typed(item, media, _cancelled=cancelled)
|
| 511 |
)
|
| 512 |
|
| 513 |
+
def _decoder_diagnostics(self) -> Any:
|
| 514 |
+
if self._decoder_graphs is not None:
|
| 515 |
+
return self._decoder_graphs.diagnostics()
|
| 516 |
+
return self._decoder_cleanup
|
| 517 |
+
|
| 518 |
def shutdown(self) -> None:
|
| 519 |
try:
|
| 520 |
+
manager, self._decoder_graphs = self._decoder_graphs, None
|
| 521 |
+
def close_graphs() -> None:
|
| 522 |
+
try:
|
| 523 |
+
manager.close()
|
| 524 |
+
finally:
|
| 525 |
+
self._decoder_cleanup = manager.diagnostics()
|
| 526 |
+
try:
|
| 527 |
+
if manager is not None:
|
| 528 |
+
if threading.get_ident() == manager.thread:
|
| 529 |
+
close_graphs()
|
| 530 |
+
else:
|
| 531 |
+
self._worker.submit(close_graphs).result()
|
| 532 |
+
finally:
|
| 533 |
+
self._worker.shutdown(wait=True, cancel_futures=True)
|
| 534 |
self._vision_cache.clear()
|
| 535 |
if self.backbone.model.get_image_features == self._image_features:
|
| 536 |
self.backbone.model.get_image_features = self._uncached_image_features
|
clairic_runtime/normalization.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Public CUDA normalization with the model's residual precision."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
| 9 |
+
"""Recover BF16 operands for an FP32 sum of two BF16 residuals."""
|
| 10 |
+
if (value.is_cuda and value.dtype == torch.float32 and value.is_contiguous()
|
| 11 |
+
and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
|
| 12 |
+
from .kernels import bf16_pair_sum
|
| 13 |
+
return bf16_pair_sum(value)
|
| 14 |
+
limit = torch.finfo(torch.bfloat16).max
|
| 15 |
+
high = value.clamp(-limit, limit).to(torch.bfloat16)
|
| 16 |
+
low = (value - high.float()).to(torch.bfloat16)
|
| 17 |
+
# Equal signed zeros keep the sign when the fused operation adds them.
|
| 18 |
+
low = torch.where(value == 0, high, low)
|
| 19 |
+
return high, low
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def rms_norm(value: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
| 23 |
+
"""Normalize CUDA BF16 states or unrounded FP32 residual sums."""
|
| 24 |
+
from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
|
| 25 |
+
|
| 26 |
+
shape = value.shape
|
| 27 |
+
flat = value.reshape(-1, shape[-1]).contiguous()
|
| 28 |
+
if value.dtype == torch.bfloat16:
|
| 29 |
+
return gemma_rmsnorm(flat, weight, eps).reshape(shape)
|
| 30 |
+
if value.dtype != torch.float32:
|
| 31 |
+
raise ValueError("Expected BF16 states or FP32 residual sums")
|
| 32 |
+
high, low = _bf16_pair_sum(flat)
|
| 33 |
+
gemma_fused_add_rmsnorm(low, high, weight, eps)
|
| 34 |
+
return low.reshape(shape)
|
clairic_runtime/readout.py
CHANGED
|
@@ -12,13 +12,23 @@ from safetensors.torch import load_file
|
|
| 12 |
from . import model as cm
|
| 13 |
|
| 14 |
|
| 15 |
-
def load_head(path: Path) -> cm.ClairicHead:
|
| 16 |
configuration = json.loads((path / "clair_head_config.json").read_text())
|
| 17 |
accepted = inspect.signature(cm.ClairicHead).parameters
|
| 18 |
head = cm.ClairicHead(
|
| 19 |
**{name: value for name, value in configuration.items() if name in accepted}
|
| 20 |
)
|
| 21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
return head.float().eval()
|
| 23 |
|
| 24 |
|
|
|
|
| 12 |
from . import model as cm
|
| 13 |
|
| 14 |
|
| 15 |
+
def load_head(path: Path, *, match_bfloat16: bool = False) -> cm.ClairicHead:
|
| 16 |
configuration = json.loads((path / "clair_head_config.json").read_text())
|
| 17 |
accepted = inspect.signature(cm.ClairicHead).parameters
|
| 18 |
head = cm.ClairicHead(
|
| 19 |
**{name: value for name, value in configuration.items() if name in accepted}
|
| 20 |
)
|
| 21 |
+
state = load_file(path / "clair_head.safetensors")
|
| 22 |
+
if match_bfloat16:
|
| 23 |
+
# Serving constructs the head in BF16 before widening it to FP32.
|
| 24 |
+
# The length bias has an explicit FP32 initialization.
|
| 25 |
+
state = {
|
| 26 |
+
name: value.bfloat16().float()
|
| 27 |
+
if value.is_floating_point() and name != "length_bias"
|
| 28 |
+
else value
|
| 29 |
+
for name, value in state.items()
|
| 30 |
+
}
|
| 31 |
+
head.load_state_dict(state, strict=True)
|
| 32 |
return head.float().eval()
|
| 33 |
|
| 34 |
|
clairic_runtime/recurrent_metadata.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Share read-only recurrent packing metadata during one inference forward."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
|
| 4 |
+
from collections.abc import Callable, Iterator
|
| 5 |
+
from contextlib import contextmanager
|
| 6 |
+
from contextvars import ContextVar
|
| 7 |
+
from typing import Any
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class _Forward:
|
| 13 |
+
def __init__(self, hook_owner: torch.nn.Module | None = None) -> None:
|
| 14 |
+
self.hook_owner = hook_owner
|
| 15 |
+
self.unversioned_mask_allowed: bool | None = None
|
| 16 |
+
self.hook_checks = 0
|
| 17 |
+
self.key: tuple[Any, ...] | None = None
|
| 18 |
+
self.mask: torch.Tensor | None = None
|
| 19 |
+
self.packed: tuple[torch.Tensor, torch.Tensor] | None = None
|
| 20 |
+
self.slots_key: tuple[Any, ...] | None = None
|
| 21 |
+
self.slots: torch.Tensor | None = None
|
| 22 |
+
self.materializations = 0
|
| 23 |
+
self.hits = 0
|
| 24 |
+
self.slots_materializations = 0
|
| 25 |
+
self.slots_hits = 0
|
| 26 |
+
|
| 27 |
+
def clear(self) -> None:
|
| 28 |
+
self.key, self.mask, self.packed = None, None, None
|
| 29 |
+
self.slots_key, self.slots = None, None
|
| 30 |
+
self.hook_owner, self.unversioned_mask_allowed = None, None
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
_current: ContextVar[_Forward | None] = ContextVar("clairic_recurrent_metadata", default=None)
|
| 34 |
+
_MAX_BYTES = 8 * 1024**2
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
@contextmanager
|
| 38 |
+
def scope(hook_owner: torch.nn.Module | None = None) -> Iterator[None]:
|
| 39 |
+
"""Drop all added references on success, cancellation, or model failure."""
|
| 40 |
+
state = _Forward(hook_owner)
|
| 41 |
+
token = _current.set(state)
|
| 42 |
+
try:
|
| 43 |
+
yield
|
| 44 |
+
finally:
|
| 45 |
+
state.clear()
|
| 46 |
+
_current.reset(token)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _eligible(mixed: torch.Tensor) -> _Forward | None:
|
| 50 |
+
state = _current.get()
|
| 51 |
+
if (state is None or not mixed.is_cuda or mixed.dtype != torch.bfloat16
|
| 52 |
+
or mixed.ndim != 3 or torch.is_grad_enabled()
|
| 53 |
+
or not torch.is_inference_mode_enabled() or torch.compiler.is_compiling()):
|
| 54 |
+
return None
|
| 55 |
+
return state
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _no_forward_hooks(owner: torch.nn.Module | None) -> bool:
|
| 59 |
+
"""Admit only a known hook-free tree and hook-free global registries."""
|
| 60 |
+
if not isinstance(owner, torch.nn.Module):
|
| 61 |
+
return False
|
| 62 |
+
registry = torch.nn.modules.module
|
| 63 |
+
for name in ("_global_forward_pre_hooks", "_global_forward_hooks"):
|
| 64 |
+
hooks = getattr(registry, name, None)
|
| 65 |
+
if hooks is None or hooks:
|
| 66 |
+
return False
|
| 67 |
+
for module in owner.modules():
|
| 68 |
+
for name in ("_forward_pre_hooks", "_forward_hooks"):
|
| 69 |
+
hooks = getattr(module, name, None)
|
| 70 |
+
if hooks is None or hooks:
|
| 71 |
+
return False
|
| 72 |
+
return True
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None,
|
| 76 |
+
original: Callable[[torch.Tensor, torch.Tensor | None],
|
| 77 |
+
tuple[torch.Tensor, torch.Tensor]]
|
| 78 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 79 |
+
"""Reuse the original helper's exact results for an unchanged current mask."""
|
| 80 |
+
state = _eligible(mixed)
|
| 81 |
+
batch, length = mixed.shape[:2]
|
| 82 |
+
required = batch * length * 8 + (2 * batch + 1) * 4
|
| 83 |
+
if state is None or required > _MAX_BYTES:
|
| 84 |
+
return original(mixed, mask)
|
| 85 |
+
stream = torch.cuda.current_stream(mixed.device).cuda_stream
|
| 86 |
+
mask_key: tuple[Any, ...] = ()
|
| 87 |
+
if mask is not None:
|
| 88 |
+
try:
|
| 89 |
+
version = mask._version
|
| 90 |
+
except RuntimeError:
|
| 91 |
+
version = None
|
| 92 |
+
if version is None:
|
| 93 |
+
if state.unversioned_mask_allowed is None:
|
| 94 |
+
state.hook_checks += 1
|
| 95 |
+
state.unversioned_mask_allowed = _no_forward_hooks(state.hook_owner)
|
| 96 |
+
if not state.unversioned_mask_allowed:
|
| 97 |
+
state.key, state.mask, state.packed = None, None, None
|
| 98 |
+
return original(mixed, mask)
|
| 99 |
+
mask_key = (id(mask), version, mask.data_ptr(), mask.storage_offset(),
|
| 100 |
+
mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride()))
|
| 101 |
+
key = (batch, length, mixed.device, stream, mask_key)
|
| 102 |
+
if state.key == key and state.mask is mask and state.packed is not None:
|
| 103 |
+
state.hits += 1
|
| 104 |
+
return state.packed
|
| 105 |
+
state.key, state.mask, state.packed = None, None, None
|
| 106 |
+
if state.slots_key != (batch, mixed.device, stream):
|
| 107 |
+
state.slots_key, state.slots = None, None
|
| 108 |
+
packed = original(mixed, mask)
|
| 109 |
+
retained = sum(value.numel() * value.element_size() for value in packed)
|
| 110 |
+
if retained + batch * 4 <= _MAX_BYTES:
|
| 111 |
+
state.key, state.mask, state.packed = key, mask, packed
|
| 112 |
+
state.materializations += 1
|
| 113 |
+
return packed
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def sequence_slots(mixed: torch.Tensor) -> torch.Tensor:
|
| 117 |
+
"""Reuse the same identity slot vector in each recurrent layer."""
|
| 118 |
+
state = _eligible(mixed)
|
| 119 |
+
batch = mixed.shape[0]
|
| 120 |
+
retained = (sum(value.numel() * value.element_size() for value in state.packed)
|
| 121 |
+
if state is not None and state.packed is not None else 0)
|
| 122 |
+
if state is None or retained + batch * 4 > _MAX_BYTES:
|
| 123 |
+
return torch.arange(batch, dtype=torch.int32, device=mixed.device)
|
| 124 |
+
key = (batch, mixed.device, torch.cuda.current_stream(mixed.device).cuda_stream)
|
| 125 |
+
if state.slots_key != key or state.slots is None:
|
| 126 |
+
state.slots = torch.arange(batch, dtype=torch.int32, device=mixed.device)
|
| 127 |
+
state.slots_key = key
|
| 128 |
+
state.slots_materializations += 1
|
| 129 |
+
else:
|
| 130 |
+
state.slots_hits += 1
|
| 131 |
+
return state.slots
|
clairic_runtime/row_omission.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Recover unresolved image rows backed by a completed row inventory."""
|
| 2 |
+
|
| 3 |
+
from copy import deepcopy
|
| 4 |
+
import json
|
| 5 |
+
from typing import Any
|
| 6 |
+
|
| 7 |
+
from . import ar, image_rows
|
| 8 |
+
from .schema import MAX_LIST_ITEMS
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def prepare(
|
| 12 |
+
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
|
| 13 |
+
) -> tuple[dict[str, Any], list[dict[str, Any]]] | None:
|
| 14 |
+
if record.get("_null_recovery") or record.get("_row_omission_recovery"):
|
| 15 |
+
return None
|
| 16 |
+
annotations = (
|
| 17 |
+
record.get("_meta", {}).get("extraction_contract", {}).get("fields", {})
|
| 18 |
+
)
|
| 19 |
+
tasks = {name: (name, ids, media) for name, ids, media in record["_ar_tasks"]}
|
| 20 |
+
if len(tasks) != len(record["_ar_tasks"]):
|
| 21 |
+
return None
|
| 22 |
+
contexts, retry_tasks = [], []
|
| 23 |
+
constraints = dict(record.get("_ar_constraints", {}))
|
| 24 |
+
for parent, annotation in annotations.items():
|
| 25 |
+
queries = annotation.get("image_row_fields")
|
| 26 |
+
spec, inventory = record["fields"].get(parent, {}), predictions.get(parent, {})
|
| 27 |
+
value = inventory.get("value")
|
| 28 |
+
if (
|
| 29 |
+
set(annotation) != {"image_row_fields"}
|
| 30 |
+
or not isinstance(queries, list)
|
| 31 |
+
or len(queries) != len(set(queries))
|
| 32 |
+
or not queries
|
| 33 |
+
or not image_rows.is_inventory(spec)
|
| 34 |
+
or spec.get("knowledge")
|
| 35 |
+
or not inventory.get("valid")
|
| 36 |
+
or not inventory.get("complete")
|
| 37 |
+
or not isinstance(value, list)
|
| 38 |
+
or not value
|
| 39 |
+
or len(value) > min(spec.get("max_items", MAX_LIST_ITEMS), len(queries))
|
| 40 |
+
or any(
|
| 41 |
+
not isinstance(row, dict)
|
| 42 |
+
or set(row) != {"label"}
|
| 43 |
+
or not isinstance(row["label"], str)
|
| 44 |
+
or not row["label"].strip()
|
| 45 |
+
for row in value
|
| 46 |
+
)
|
| 47 |
+
or any(child not in predictions for child in queries)
|
| 48 |
+
):
|
| 49 |
+
continue
|
| 50 |
+
properties = spec.get("items", {}).get("properties", {})
|
| 51 |
+
text_keys = [
|
| 52 |
+
key
|
| 53 |
+
for key, child in properties.items()
|
| 54 |
+
if child.get("type") == "text" and not child.get("knowledge")
|
| 55 |
+
]
|
| 56 |
+
if not text_keys:
|
| 57 |
+
continue
|
| 58 |
+
labels = [row["label"] for row in value]
|
| 59 |
+
proposals = [predictions[child] for child in queries]
|
| 60 |
+
selected, conflicts = image_rows._select(labels, proposals, text_keys)
|
| 61 |
+
if not conflicts or any(
|
| 62 |
+
row is not None and (not row.get("valid") or not row.get("complete"))
|
| 63 |
+
for row in selected
|
| 64 |
+
):
|
| 65 |
+
continue
|
| 66 |
+
children, local_constraints = [], {}
|
| 67 |
+
for index in conflicts:
|
| 68 |
+
child = queries[index]
|
| 69 |
+
row, child_spec = predictions[child], record["fields"].get(child, {})
|
| 70 |
+
row_value = row.get("value")
|
| 71 |
+
task = tasks.get(child)
|
| 72 |
+
native = constraints.get(child, {})
|
| 73 |
+
if (
|
| 74 |
+
annotations.get(child) != {"parent": parent, "index": index + 1}
|
| 75 |
+
or child_spec.get("type") != "object"
|
| 76 |
+
or not child_spec.get("_image_row_proposal")
|
| 77 |
+
or child_spec.get("knowledge")
|
| 78 |
+
or not row.get("valid")
|
| 79 |
+
or not row.get("complete")
|
| 80 |
+
or not isinstance(row_value, dict)
|
| 81 |
+
or not row_value
|
| 82 |
+
or any(cell is not None for cell in row_value.values())
|
| 83 |
+
or any(row is previous for previous in selected if previous is not None)
|
| 84 |
+
or task is None
|
| 85 |
+
or task[2] is None
|
| 86 |
+
or set(native) != {"json_schema"}
|
| 87 |
+
):
|
| 88 |
+
break
|
| 89 |
+
schema = json.loads(native["json_schema"])
|
| 90 |
+
if schema != ar.schema(child_spec) or not set(text_keys) <= set(
|
| 91 |
+
schema.get("properties", {})
|
| 92 |
+
):
|
| 93 |
+
break
|
| 94 |
+
# A completed inventory requires a matching textual cell; the
|
| 95 |
+
# original selector remains the acceptance test for that label.
|
| 96 |
+
# Full branches retain required fields with the native grammar compiler.
|
| 97 |
+
branches = []
|
| 98 |
+
for key in text_keys:
|
| 99 |
+
branch = deepcopy(schema)
|
| 100 |
+
branch["properties"][key]["type"] = "string"
|
| 101 |
+
branches.append(branch)
|
| 102 |
+
local_constraints[child] = {"json_schema": json.dumps({"anyOf": branches})}
|
| 103 |
+
children.append(child)
|
| 104 |
+
else:
|
| 105 |
+
constraints.update(local_constraints)
|
| 106 |
+
retry_tasks.extend(tasks[child] for child in children)
|
| 107 |
+
contexts.append(
|
| 108 |
+
{
|
| 109 |
+
"parent": parent,
|
| 110 |
+
"queries": queries,
|
| 111 |
+
"labels": labels,
|
| 112 |
+
"text_keys": text_keys,
|
| 113 |
+
"selected": selected,
|
| 114 |
+
"conflicts": conflicts,
|
| 115 |
+
"children": children,
|
| 116 |
+
}
|
| 117 |
+
)
|
| 118 |
+
if not retry_tasks:
|
| 119 |
+
return None
|
| 120 |
+
repair = {
|
| 121 |
+
**record,
|
| 122 |
+
"_ar_tasks": retry_tasks,
|
| 123 |
+
"_ar_constraints": constraints,
|
| 124 |
+
"_ar_prefix": None,
|
| 125 |
+
"_ar_group_prefixes": [],
|
| 126 |
+
"_ar_media_groups": [],
|
| 127 |
+
"_null_recovery": True,
|
| 128 |
+
"_row_omission_recovery": True,
|
| 129 |
+
}
|
| 130 |
+
return repair, contexts
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def merge(
|
| 134 |
+
record: dict[str, Any],
|
| 135 |
+
predictions: dict[str, dict[str, Any]],
|
| 136 |
+
alternatives: dict[str, dict[str, Any]],
|
| 137 |
+
plan: tuple[dict[str, Any], list[dict[str, Any]]],
|
| 138 |
+
) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]:
|
| 139 |
+
repair, contexts = plan
|
| 140 |
+
lengths = {name: len(ids) for name, ids, _ in repair["_ar_tasks"]}
|
| 141 |
+
result = dict(predictions)
|
| 142 |
+
evidence = []
|
| 143 |
+
for context in contexts:
|
| 144 |
+
candidates = {}
|
| 145 |
+
for child in context["children"]:
|
| 146 |
+
original, alternative = predictions[child], alternatives.get(child, {})
|
| 147 |
+
account = {
|
| 148 |
+
"tokens": original["tokens"] + alternative.get("tokens", 0),
|
| 149 |
+
"retry_input_tokens": original.get("retry_input_tokens", 0)
|
| 150 |
+
+ lengths[child],
|
| 151 |
+
}
|
| 152 |
+
result[child] = {**original, **account}
|
| 153 |
+
value = alternative.get("value")
|
| 154 |
+
if (
|
| 155 |
+
alternative.get("valid")
|
| 156 |
+
and alternative.get("complete")
|
| 157 |
+
and isinstance(value, dict)
|
| 158 |
+
and any(cell is not None for cell in value.values())
|
| 159 |
+
):
|
| 160 |
+
candidates[child] = {
|
| 161 |
+
**alternative,
|
| 162 |
+
**account,
|
| 163 |
+
"image_row_omission_recovered": True,
|
| 164 |
+
}
|
| 165 |
+
trial = {**result, **candidates}
|
| 166 |
+
selected, conflicts = image_rows._select(
|
| 167 |
+
context["labels"],
|
| 168 |
+
[trial[name] for name in context["queries"]],
|
| 169 |
+
context["text_keys"],
|
| 170 |
+
)
|
| 171 |
+
unchanged = all(
|
| 172 |
+
previous is None or selected[index] is previous
|
| 173 |
+
for index, previous in enumerate(context["selected"])
|
| 174 |
+
)
|
| 175 |
+
accepted = (
|
| 176 |
+
not conflicts and unchanged and len(candidates) == len(context["children"])
|
| 177 |
+
)
|
| 178 |
+
if accepted:
|
| 179 |
+
result.update(candidates)
|
| 180 |
+
evidence.append(
|
| 181 |
+
{
|
| 182 |
+
"parent": context["parent"],
|
| 183 |
+
"original_conflicts": context["conflicts"],
|
| 184 |
+
"retry_children": context["children"],
|
| 185 |
+
"conflicts_after_retry": conflicts,
|
| 186 |
+
"previously_matched_proposals_preserved_exactly": unchanged,
|
| 187 |
+
"accepted": accepted,
|
| 188 |
+
}
|
| 189 |
+
)
|
| 190 |
+
return result, evidence
|
requirements.txt
CHANGED
|
@@ -1,9 +1,15 @@
|
|
| 1 |
-
torch==2.
|
| 2 |
-
torchvision==0.
|
| 3 |
transformers==5.10.2
|
| 4 |
kernels==0.13.0
|
| 5 |
tomlkit==0.13.3
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
flash-linear-attention==0.6.0
|
| 8 |
fla-core==0.6.0
|
| 9 |
einops==0.8.1
|
|
|
|
| 1 |
+
torch==2.13.0
|
| 2 |
+
torchvision==0.28.0
|
| 3 |
transformers==5.10.2
|
| 4 |
kernels==0.13.0
|
| 5 |
tomlkit==0.13.3
|
| 6 |
+
sgl-deep-gemm==0.1.5.post3
|
| 7 |
+
apache-tvm-ffi==0.1.11
|
| 8 |
+
flashinfer-python==0.6.17; sys_platform == "linux" and platform_machine == "x86_64"
|
| 9 |
+
flashinfer-cubin @ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.17/flashinfer_cubin-0.6.17-py3-none-any.whl#sha256=771f037a828cdf3f15db1c3784574197546d1b09735fb9058f384bdd12010afa ; sys_platform == "linux" and platform_machine == "x86_64"
|
| 10 |
+
flashinfer-jit-cache @ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.17/flashinfer_jit_cache-0.6.17+cu130-cp39-abi3-manylinux_2_28_x86_64.whl#sha256=e7dd1deecc7f348d41b1eb56217d945d4f1630f447ff9b0b9e7919b0e07c01f0 ; sys_platform == "linux" and platform_machine == "x86_64"
|
| 11 |
+
nvidia-cutlass-dsl[cu13]==4.6.2; sys_platform == "linux" and platform_machine == "x86_64"
|
| 12 |
+
triton==3.7.1; sys_platform == "linux" and platform_machine == "x86_64"
|
| 13 |
flash-linear-attention==0.6.0
|
| 14 |
fla-core==0.6.0
|
| 15 |
einops==0.8.1
|