areneau's picture
Update Clairic Transformers runtime
24191a8 verified
Raw History Blame Contribute Delete
24 kB
# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
# Copyright 2026 Cloudflare, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
"""Numerical operations for the Transformers Clairic backbone."""
from __future__ import annotations
from collections.abc import Callable
from types import MethodType
from typing import Any
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils._python_dispatch import TorchDispatchMode
from transformers.cache_utils import Cache
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from transformers.models.qwen3_5.modeling_qwen3_5 import (
Qwen3_5Attention, Qwen3_5DecoderLayer, Qwen3_5ForConditionalGeneration,
Qwen3_5GatedDeltaNet, Qwen3_5MLP, Qwen3_5RMSNorm,
Qwen3_5TextRotaryEmbedding, apply_mask_to_padding_states,
eager_attention_forward, torch_causal_conv1d_update,
)
from .attention_metadata import DecodeMetadata
from .recurrent_metadata import scope as recurrent_metadata_scope
def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
raise ValueError("Expected aligned matrices with equal dtype")
if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
and a.is_contiguous() and b.transpose(0, 1).is_contiguous()
and b.shape[1] >= 16):
from deep_gemm import bf16_gemm_nn
output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype)
bf16_gemm_nn(a, b, output)
if bias is not None:
output += bias
return output
from .kernels import _matmul_persistent_triton
return _matmul_persistent_triton(a, b, bias)
def _first_tensor(values: Any) -> torch.Tensor | None:
if isinstance(values, torch.Tensor):
return values
if isinstance(values, (list, tuple)):
return next((found for value in values if (found := _first_tensor(value)) is not None), None)
return None
class ReadoutMath(TorchDispatchMode):
"""Apply CUDA readout operations within the current worker call."""
def __torch_dispatch__(self: ReadoutMath, function: Any, types: Any, args: tuple[Any, ...] = (),
kwargs: dict[str, Any] | None = None) -> Any:
kwargs = {} if kwargs is None else kwargs
first = _first_tensor(args)
eligible = first is not None and first.is_cuda and first.dtype in (torch.bfloat16, torch.float32)
if eligible and function in (torch.ops.aten.linear.default, torch.ops.aten.matmul.default,
torch.ops.aten.einsum.default,
torch.ops.aten.scaled_dot_product_attention.default,
torch.ops.aten._scaled_dot_product_attention_math.default):
with self:
result = function.decompose(*args, **kwargs)
if result is NotImplemented:
raise RuntimeError(f"Missing matrix decomposition: {function}")
return result
if eligible and function is torch.ops.aten._native_multi_head_attention.default:
with self:
return function._op_dk(torch._C.DispatchKey.CUDA, *args, **kwargs)
if eligible and function is torch.ops.aten.mm.default:
return _matmul(args[0], args[1])
if eligible and function is torch.ops.aten.addmm.default:
if kwargs.get("alpha", 1) != 1 or kwargs.get("beta", 1) != 1:
raise ValueError("Expected unit addmm alpha and beta")
return _matmul(args[1], args[2], args[0])
if eligible and function is torch.ops.aten.mm.dtype:
return _matmul(args[0].contiguous(), args[1].contiguous()).to(kwargs["out_dtype"])
if eligible and function is torch.ops.aten.bmm.default:
from .kernels import bmm_batch_invariant
return bmm_batch_invariant(args[0], args[1])
if eligible and function is torch.ops.aten.mean.dim:
from .kernels import mean_batch_invariant
return mean_batch_invariant(args[0], args[1], args[2] if len(args) > 2 else False,
dtype=kwargs.get("dtype"))
return function(*args, **kwargs)
def _linear_forward(original: Callable[..., Any], lm_head: bool,
transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]:
cache: dict[str, Any] = {}
if lm_head and transpose_caches is not None:
transpose_caches.append(cache)
def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
or input.dtype != self.weight.dtype):
cache.clear()
return original(input)
shape = input.shape[:-1]
flat = input.reshape(-1, input.shape[-1])
if lm_head:
weight = self.weight
version = None
if not torch.is_grad_enabled() and not torch.compiler.is_compiling():
try:
version = weight._version
except RuntimeError:
pass
if version is None:
cache.clear()
transposed = weight.T.contiguous()
else:
stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None
key = (id(weight), version, weight.data_ptr(), weight.storage_offset(),
weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream)
if cache.get("weight") is not weight or cache.get("key") != key:
cache.clear()
cache["transposed"] = weight.T.contiguous()
cache["key"] = key
cache["weight"] = weight
transposed = cache["transposed"]
output = _matmul(flat.contiguous(), transposed).float()
else:
output = _matmul(flat, self.weight.T, self.bias)
return output.reshape(*shape, self.weight.shape[0])
return forward
def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32):
from .normalization import rms_norm
return rms_norm(x, self.weight, self.eps)
value = x.float()
value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
return (value * (1.0 + self.weight.float())).to(self.weight.dtype)
def _residual_sum(first: torch.Tensor, second: torch.Tensor,
dtype: torch.dtype) -> torch.Tensor:
if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda
and first.device == second.device and first.shape == second.shape
and first.dtype in (torch.bfloat16, torch.float32)
and second.dtype == torch.bfloat16
and first.is_contiguous() and second.is_contiguous()
and first.numel() <= 2**31 - 1
and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
from .kernels import residual_sum
return residual_sum(first, second)
return first.to(dtype).float() + second.float()
def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
attention_mask: torch.Tensor | None = None,
position_ids: torch.Tensor | None = None, past_key_values: Cache | None = None,
**kwargs: Any) -> torch.Tensor:
dtype = self.input_layernorm.weight.dtype
residual = hidden_states.to(dtype)
states = self.input_layernorm(hidden_states)
if self.layer_type == "linear_attention":
states = self.linear_attn(hidden_states=states, cache_params=past_key_values,
attention_mask=attention_mask, **kwargs)
else:
states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
position_ids=position_ids, past_key_values=past_key_values,
position_embeddings=position_embeddings, **kwargs)
merged = _residual_sum(residual, states, dtype)
states = self.post_attention_layernorm(merged)
return _residual_sum(merged, self.mlp(states), dtype)
def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
gate, up = self.gate_proj(x), self.up_proj(x)
if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16
and gate.is_contiguous() and up.is_contiguous()):
from .kernels import silu_mul
return self.down_proj(silu_mul(gate, up))
return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))
def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor,
sin: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
dim = cos.shape[-1]
half = dim // 2
c, s = cos[:, None, :, :half], sin[:, None, :, :half]
def apply(value: torch.Tensor) -> torch.Tensor:
first, second = value[..., :half].float(), value[..., half:dim].float()
rotated = torch.cat((first * c - second * s, second * c + first * s), -1)
return torch.cat((rotated.to(value.dtype), value[..., dim:]), -1)
return apply(q), apply(k)
def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
*, metadata: DecodeMetadata | None = None,
**kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
input_shape = hidden_states.shape[:-1]
hidden_shape = (*input_shape, -1, self.head_dim)
q_gate, raw_key = self.q_proj(hidden_states), self.k_proj(hidden_states)
if hidden_states.is_cuda:
from .kernels import dense_qk
query, key, gate = dense_qk(q_gate, raw_key, self.q_norm.weight,
self.k_norm.weight, *position_embeddings, self.q_norm.eps)
else:
query, gate = torch.chunk(q_gate.view(*input_shape, -1, self.head_dim * 2), 2, dim=-1)
gate = gate.reshape(*input_shape, -1)
query = self.q_norm(query.view(hidden_shape)).transpose(1, 2)
key = self.k_norm(raw_key.view(hidden_shape)).transpose(1, 2)
query, key = _rotate(query, key, *position_embeddings)
value = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
if past_key_values is not None:
key, value = past_key_values.update(key, value, self.layer_idx)
interface = ALL_ATTENTION_FUNCTIONS.get_interface(
self.config._attn_implementation, eager_attention_forward)
if metadata is not None:
output, weights = metadata.attend(interface, self, query, key, value, attention_mask,
dropout=0.0 if not self.training else self.attention_dropout,
scaling=self.scaling, **kwargs)
else:
output, weights = interface(self, query, key, value, attention_mask,
dropout=0.0 if not self.training else self.attention_dropout,
scaling=self.scaling, **kwargs)
output = output.reshape(*input_shape, -1).contiguous()
if output.is_cuda and output.dtype == torch.bfloat16:
from .kernels import fused_sigmoid_mul
output = fused_sigmoid_mul(output, gate, inplace=True)
else:
output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype)
return self.o_proj(output), weights
def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor,
cache_params: Cache | None = None, attention_mask: torch.Tensor | None = None,
**kwargs: Any) -> torch.Tensor:
hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
batch, length, _ = hidden_states.shape
cached = cache_params is not None and cache_params.has_previous_state(self.layer_idx)
if cached:
conv_state = cache_params.layers[self.layer_idx].conv_states
recurrent_state = cache_params.layers[self.layer_idx].recurrent_states
mixed = self.in_proj_qkv(hidden_states).transpose(1, 2)
z = self.in_proj_z(hidden_states).reshape(batch, length, -1, self.head_v_dim)
b, a = self.in_proj_b(hidden_states), self.in_proj_a(hidden_states)
if cached and length == 1:
mixed = self.causal_conv1d_update(mixed, conv_state, self.conv1d.weight.squeeze(1),
self.conv1d.bias, self.activation)
else:
if cached:
mixed = torch.cat((conv_state, mixed), -1)
if cache_params is not None:
state = F.pad(mixed, (self.conv_kernel_size - mixed.shape[-1], 0))
cache_params.update_conv_state(state, self.layer_idx)
if self.causal_conv1d_fn is not None:
mixed = self.causal_conv1d_fn(x=mixed, weight=self.conv1d.weight.squeeze(1),
bias=self.conv1d.bias, activation=self.activation, seq_idx=kwargs.get("seq_idx"))
else:
mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
if cached:
mixed = mixed[:, :, -length:]
from .gdn.adapter import decode, prefill, store_state
if cached and length == 1:
output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
recurrent_state)
else:
output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim,
recurrent_state if cached else None, attention_mask)
if cache_params is not None:
store_state(cache_params, self.layer_idx, state)
output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
return self.out_proj(output.reshape(batch, length, -1))
def _gated_norm(self: Any, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
epsilon = getattr(self, "eps", getattr(self, "variance_epsilon", None))
if (x.ndim != 2 or z.shape != x.shape or self.weight.shape != (128,)
or x.shape[-1] != 128 or x.shape[0] < 1 or epsilon != 1e-6
or x.dtype != z.dtype or self.weight.dtype != x.dtype
or any(v.device != x.device or v.stride(-1) != 1 for v in (x, z, self.weight))):
raise ValueError("Unsupported gated normalization inputs")
from .kernels import gated_norm
return gated_norm(x, z, self.weight, epsilon)
def _nonoverlapping(value: torch.Tensor) -> None:
span = 1
for stride, size in sorted((s, n) for s, n in zip(value.stride(), value.shape, strict=True) if n > 1):
if stride < span:
raise ValueError("Overlapping convolution tensor")
span += (size - 1) * stride
def _validate_conv(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None,
activation: str | None) -> None:
if (x.ndim != 3 or weight.ndim != 2 or tuple(weight.shape) != (x.shape[1], 4)
or min(x.shape) < 1 or weight.dtype != x.dtype or weight.device != x.device
or not weight.is_contiguous()):
raise ValueError("Expected nonempty input[B,D,T] and matching contiguous weight[D,4]")
if bias is not None and (tuple(bias.shape) != (x.shape[1],) or bias.dtype != x.dtype
or bias.device != x.device or not bias.is_contiguous()):
raise ValueError("Expected a matching contiguous bias[D]")
if activation not in (None, "silu", "swish"):
raise ValueError("Unsupported convolution activation")
_nonoverlapping(x)
def _prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None,
activation: str | None = None, seq_idx: torch.Tensor | None = None
) -> torch.Tensor:
if seq_idx is not None:
raise ValueError("Packed convolution sequence indices are unsupported")
if not x.is_cuda or x.dtype != torch.bfloat16:
output = F.conv1d(x, weight.unsqueeze(1), bias, padding=weight.shape[-1] - 1,
groups=x.shape[1])[:, :, :x.shape[2]]
return F.silu(output) if activation is not None else output
_validate_conv(x, weight, bias, activation)
from .kernels import prefill
return prefill(x, weight, bias, activation)
def _update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor,
bias: torch.Tensor | None = None, activation: str | None = None) -> torch.Tensor:
if not x.is_cuda or x.dtype != torch.bfloat16:
return torch_causal_conv1d_update(x, state, weight, bias, activation)
_validate_conv(x, weight, bias, activation)
if (x.shape[2] != 1 or tuple(state.shape) != (x.shape[0], x.shape[1], 4)
or state.dtype != x.dtype or state.device != x.device):
raise ValueError("Expected singleton input and matching live cache[B,D,4]")
_nonoverlapping(state)
pointer = state.untyped_storage().data_ptr()
if any(v.untyped_storage().data_ptr() == pointer for v in (x, weight, bias) if v is not None):
raise ValueError("Convolution cache must not alias input, weight or bias")
from .kernels import update
return update(x, state, weight, bias, activation)
def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None = None
) -> Callable[[], None]:
"""Configure this backbone and return an idempotent cleanup function."""
def native_cleanup() -> None:
return None
language = backbone.model.language_model
if language.embed_tokens.weight.dtype != torch.bfloat16:
return native_cleanup
rotary = language.rotary_emb
rope = rotary.config.rope_parameters
if (rope.get("rope_type") != "default" or rope.get("partial_rotary_factor") != 0.25
or rope.get("rope_theta") != 10000000.0 or rotary.config.head_dim != 256
or rope.get("mrope_interleaved") is not True
or list(rope.get("mrope_section", ())) != [11, 11, 10]
or language.config.hidden_act != "silu"):
raise ValueError("Unsupported Clairic rotary configuration")
forwards: list[tuple[nn.Module, bool, Any, Any]] = []
convolutions: list[tuple[Any, Any, Any]] = []
caches: dict[torch.device, torch.Tensor] = {}
transpose_caches: list[dict[str, Any]] = []
attention_metadata = DecodeMetadata()
closed = False
def restore() -> None:
nonlocal closed
if closed:
return
closed = True
attention_metadata.close()
for module, original_prefill, original_update in reversed(convolutions):
module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
for module, had_override, original, installed in reversed(forwards):
if module.forward == installed:
if had_override:
module.forward = original
else:
del module.forward
caches.clear()
for cache in transpose_caches:
cache.clear()
transpose_caches.clear()
def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
original = module.forward
def forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
value = args[0] if args else kwargs[argument]
dtype = parameter.dtype if parameter is not None else value.dtype
active = dtype == torch.bfloat16 and (not cuda_only or value.is_cuda)
return function(self, *args, **kwargs) if active else original(*args, **kwargs)
installed = MethodType(forward, module)
forwards.append((module, "forward" in module.__dict__, original, installed))
module.forward = installed
def text_rotary(self: Qwen3_5TextRotaryEmbedding, x: torch.Tensor,
position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if x.device not in caches:
dim = int(self.config.head_dim * rope["partial_rotary_factor"])
with torch.no_grad(), torch.autocast(device_type=x.device.type, enabled=False):
inv = 1.0 / (rope["rope_theta"] ** (
torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim))
positions = torch.arange(self.config.max_position_embeddings,
dtype=torch.float32, device=x.device)
frequencies = torch.einsum("i,j -> ij", positions, inv)
caches[x.device] = torch.cat((frequencies.cos(), frequencies.sin()), -1)
positions = position_ids[0] if position_ids.ndim == 3 else position_ids
cos, sin = caches[x.device][positions].chunk(2, -1)
return torch.cat((cos, cos), -1), torch.cat((sin, sin), -1)
try:
for module in language.modules():
if isinstance(module, Qwen3_5RMSNorm):
bind(module, _norm, "x", module.weight)
elif isinstance(module, Qwen3_5DecoderLayer):
bind(module, _decoder, parameter=module.input_layernorm.weight)
elif isinstance(module, Qwen3_5MLP):
bind(module, _mlp, "x", module.gate_proj.weight)
elif isinstance(module, Qwen3_5Attention):
def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any:
return _dense(self, *args, metadata=attention_metadata, **kwargs)
bind(module, dense, parameter=module.q_proj.weight)
elif isinstance(module, Qwen3_5GatedDeltaNet):
bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True)
bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
bind(rotary, text_rotary, "x")
original_language_forward = language.forward
def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self):
return original_language_forward(*args, **kwargs)
installed_language = MethodType(language_forward, language)
forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language))
language.forward = installed_language
lm_head = backbone.get_output_embeddings()
roots = (backbone,) if head is None else (backbone, head)
seen: set[int] = set()
for root in roots:
for module in root.modules():
if id(module) in seen or (type(module) is not nn.Linear and module is not lm_head):
continue
seen.add(id(module))
if module.weight.dtype not in (torch.bfloat16, torch.float32):
continue
original = module.forward
installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module)
forwards.append((module, "forward" in module.__dict__, original, installed))
module.forward = installed
except BaseException:
restore()
raise
return restore