Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/kernels.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 33.7 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/kernels.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/kernels.py
-
curl -L -o kernels.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/kernels.py
33.7 kB
| # SPDX-License-Identifier: Apache-2.0 | |
| # SPDX-FileCopyrightText: Copyright contributors to the vLLM project | |
| # Copyright (c) 2024, Tri Dao. | |
| # Copyright 2026 Cloudflare, Inc. | |
| """CUDA kernels for attention and causal convolution.""" | |
| from typing import Any | |
| import torch | |
| import triton | |
| import triton.language as tl | |
| def _prefill(X: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor, | |
| T: tl.int32, D: tl.constexpr, | |
| XB: tl.int64, XD: tl.int64, XT: tl.int64, | |
| OB: tl.int64, OD: tl.int64, OT: tl.int64, | |
| HAS_BIAS: tl.constexpr, SILU: tl.constexpr, | |
| BT: tl.constexpr, BD: tl.constexpr) -> None: | |
| batch = tl.program_id(1) | |
| token = tl.program_id(0) * BT + tl.arange(0, BT) | |
| channel = tl.program_id(2) * BD + tl.arange(0, BD) | |
| if HAS_BIAS: | |
| initial = tl.load(Bias + channel, channel < D, other=0).to(tl.float32) | |
| else: | |
| initial = tl.full((BD,), 0, tl.float32) | |
| acc = tl.broadcast_to(initial[None, :], (BT, BD)) | |
| for tap in tl.static_range(4): | |
| position = token + tap - 3 | |
| value = tl.load(X + batch * XB + channel[None, :] * XD + position[:, None] * XT, | |
| (position[:, None] >= 0) & (position[:, None] < T) | |
| & (channel[None, :] < D), other=0) | |
| weight = tl.load(W + channel * 4 + tap, channel < D, other=0) | |
| acc += value * weight[None, :] | |
| if SILU: | |
| acc = acc / (1 + tl.exp(-acc)) | |
| tl.store(Out + batch * OB + channel[None, :] * OD + token[:, None] * OT, | |
| acc, (token[:, None] < T) & (channel[None, :] < D)) | |
| def _update(X: tl.tensor, State: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor, | |
| D: tl.constexpr, XB: tl.int64, XD: tl.int64, | |
| SB: tl.int64, SD: tl.int64, ST: tl.int64, | |
| OB: tl.int64, OD: tl.int64, | |
| HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BD: tl.constexpr) -> None: | |
| batch = tl.program_id(0) | |
| channel = tl.program_id(1) * BD + tl.arange(0, BD) | |
| mask = channel < D | |
| base = State + batch * SB + channel * SD | |
| x0 = tl.load(base + ST, mask, other=0) | |
| x1 = tl.load(base + 2 * ST, mask, other=0) | |
| x2 = tl.load(base + 3 * ST, mask, other=0) | |
| x3 = tl.load(X + batch * XB + channel * XD, mask, other=0) | |
| if HAS_BIAS: | |
| acc = tl.load(Bias + channel, mask, other=0).to(tl.float32) | |
| else: | |
| acc = tl.full((BD,), 0, tl.float32) | |
| for tap in tl.static_range(4): | |
| if tap == 0: | |
| value = x0 | |
| elif tap == 1: | |
| value = x1 | |
| elif tap == 2: | |
| value = x2 | |
| else: | |
| value = x3 | |
| weight = tl.load(W + channel * 4 + tap, mask, other=0) | |
| acc += value * weight | |
| if SILU: | |
| acc = acc / (1 + tl.exp(-acc)) | |
| tl.store(Out + batch * OB + channel * OD, acc, mask) | |
| # Preserve the four-token cache owned by Transformers. | |
| tl.store(base, x0, mask) | |
| tl.store(base + ST, x1, mask) | |
| tl.store(base + 2 * ST, x2, mask) | |
| tl.store(base + 3 * ST, x3, mask) | |
| def prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, | |
| activation: str | None) -> torch.Tensor: | |
| output = torch.empty_like(x) | |
| batch, channels, length = x.shape | |
| _prefill[(triton.cdiv(length, 8), batch, triton.cdiv(channels, 256))]( | |
| x, weight, bias, output, length, channels, *x.stride(), *output.stride(), | |
| HAS_BIAS=bias is not None, SILU=activation is not None, BT=8, BD=256, | |
| num_warps=4, num_stages=2, | |
| ) | |
| return output | |
| def update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor, | |
| bias: torch.Tensor | None, activation: str | None) -> torch.Tensor: | |
| output = torch.empty_like(x) | |
| _update[(x.shape[0], triton.cdiv(x.shape[1], 256))]( | |
| x, state, weight, bias, output, x.shape[1], *x.stride()[:2], | |
| *state.stride(), *output.stride()[:2], HAS_BIAS=bias is not None, | |
| SILU=activation is not None, BD=256, num_warps=4, | |
| ) | |
| return output | |
| def _dense_qk_kernel(QG: tl.tensor, K: tl.tensor, QO: tl.tensor, KO: tl.tensor, GO: tl.tensor, | |
| QW: tl.tensor, KW: tl.tensor, C: tl.tensor, S: tl.tensor, | |
| QGT: tl.int64, KT: tl.int64, QT: tl.int64, KOT: tl.int64, GT: tl.int64, | |
| CT: tl.int64, ST: tl.int64, | |
| NQ: tl.constexpr, NKV: tl.constexpr, D: tl.constexpr, R: tl.constexpr, | |
| HALF: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_BLOCK: tl.constexpr, | |
| EPS: tl.constexpr) -> None: | |
| token, head = tl.program_id(0), tl.program_id(1) | |
| is_k = head >= NQ | |
| local_head = tl.where(is_k, head - NQ, head) | |
| if is_k: | |
| in_base = K + token * KT + local_head * D | |
| w_ptr = KW | |
| out_base = KO + token * KOT + local_head * D | |
| else: | |
| in_base = QG + token * QGT + local_head * 2 * D | |
| w_ptr = QW | |
| out_base = QO + token * QT + local_head * D | |
| head_offs = tl.arange(0, HEAD_BLOCK) | |
| head_mask = head_offs < D | |
| x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32) | |
| w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) | |
| var = tl.sum(x * x, axis=0) / D | |
| inv_rms = tl.rsqrt(var + EPS) | |
| x_norm = (x * inv_rms * (w + 1.0)).to(tl.bfloat16).to(tl.float32) | |
| pass_mask = head_mask & (head_offs >= R) | |
| tl.store(out_base + head_offs, x_norm, mask=pass_mask) | |
| rot_offs = tl.arange(0, ROT_BLOCK) | |
| rot_mask = rot_offs < HALF | |
| xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| xr2 = tl.load(in_base + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| wr2 = tl.load(w_ptr + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(tl.bfloat16).to(tl.float32) | |
| xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(tl.bfloat16).to(tl.float32) | |
| cos = tl.load(C + token * CT + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| sin = tl.load(S + token * ST + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) | |
| tl.store(out_base + rot_offs, xr1 * cos - xr2 * sin, mask=rot_mask) | |
| tl.store(out_base + HALF + rot_offs, xr2 * cos + xr1 * sin, mask=rot_mask) | |
| gate_in = in_base + D | |
| if not is_k: | |
| gate_out = GO + token * GT + local_head * D | |
| gate = tl.load(gate_in + head_offs, mask=head_mask, other=0.0) | |
| tl.store(gate_out + head_offs, gate, mask=head_mask) | |
| def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor, | |
| k_weight: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, | |
| eps: float) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| batch, length, q_width = q_gate.shape | |
| dim, rotary_dim = q_weight.numel(), cos.shape[-1] | |
| q_heads, kv_heads = q_width // (2 * dim), key.shape[-1] // dim | |
| tokens = batch * length | |
| qg, k = q_gate.view(tokens, -1), key.view(tokens, -1) | |
| c, s = cos.view(tokens, rotary_dim), sin.view(tokens, rotary_dim) | |
| q_out = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype) | |
| k_out = torch.empty(tokens, kv_heads * dim, device=key.device, dtype=key.dtype) | |
| gate = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype) | |
| _dense_qk_kernel[(tokens, q_heads + kv_heads)]( | |
| qg, k, q_out, k_out, gate, q_weight, k_weight, c, s, | |
| qg.stride(0), k.stride(0), q_out.stride(0), k_out.stride(0), gate.stride(0), | |
| c.stride(0), s.stride(0), NQ=q_heads, NKV=kv_heads, D=dim, R=rotary_dim, | |
| HALF=rotary_dim // 2, HEAD_BLOCK=triton.next_power_of_2(dim), | |
| ROT_BLOCK=triton.next_power_of_2(rotary_dim // 2), EPS=eps, num_warps=4) | |
| return (q_out.view(batch, length, q_heads, dim).transpose(1, 2), | |
| k_out.view(batch, length, kv_heads, dim).transpose(1, 2), | |
| gate.view(batch, length, q_heads * dim)) | |
| def _layer_norm_fwd_1pass_kernel( | |
| X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor, | |
| Mean: tl.tensor, Rstd: tl.tensor, | |
| stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64, | |
| M: tl.int32, N: tl.constexpr, eps: tl.float32, | |
| BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr, | |
| HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr, | |
| IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr, | |
| USE_GDC: tl.constexpr = False, | |
| ) -> None: | |
| if USE_GDC: | |
| tl.extra.cuda.gdc_wait() | |
| row_start = tl.program_id(0) * ROWS_PER_BLOCK | |
| group = tl.program_id(1) | |
| rows = row_start + tl.arange(0, ROWS_PER_BLOCK) | |
| cols = tl.arange(0, BLOCK_N) | |
| row_offsets = rows[:, None] * stride_x_row | |
| col_offsets = cols[None, :] + group * N | |
| X_base = X + row_offsets + col_offsets | |
| Y_base = Y + rows[:, None] * stride_y_row + col_offsets | |
| row_mask = rows[:, None] < M | |
| col_mask = cols[None, :] < N | |
| mask = row_mask & col_mask | |
| x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32) | |
| if HAS_Z and (not NORM_BEFORE_GATE): | |
| Z_base = Z + rows[:, None] * stride_z_row + col_offsets | |
| z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) | |
| if ACTIVATION == 'swish' or ACTIVATION == 'silu': | |
| x *= z * tl.sigmoid(z) | |
| elif ACTIVATION == 'sigmoid': | |
| x *= tl.sigmoid(z) | |
| if not IS_RMS_NORM: | |
| mean = tl.sum(x, axis=1) / N | |
| mean_offsets = group * M + rows | |
| mean_mask = rows < M | |
| tl.store(Mean + mean_offsets, mean, mask=mean_mask) | |
| xbar = tl.where(mask, x - mean[:, None], 0.0) | |
| var = tl.sum(xbar * xbar, axis=1) / N | |
| else: | |
| xbar = tl.where(mask, x, 0.0) | |
| var = tl.sum(xbar * xbar, axis=1) / N | |
| mean = 0.0 | |
| rstd = tl.rsqrt(var + eps) | |
| rstd_offsets = group * M + rows | |
| rstd_mask = rows < M | |
| tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask) | |
| w_offsets = cols + group * N | |
| w_mask = cols < N | |
| w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32) | |
| if HAS_BIAS: | |
| b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32) | |
| if not IS_RMS_NORM: | |
| x_hat = (x - mean[:, None]) * rstd[:, None] | |
| else: | |
| x_hat = x * rstd[:, None] | |
| y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :] | |
| if HAS_Z and NORM_BEFORE_GATE: | |
| Z_base = Z + rows[:, None] * stride_z_row + col_offsets | |
| z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) | |
| if ACTIVATION == 'swish' or ACTIVATION == 'silu': | |
| y *= z * tl.sigmoid(z) | |
| elif ACTIVATION == 'sigmoid': | |
| y *= tl.sigmoid(z) | |
| tl.store(Y_base, y, mask=mask) | |
| if USE_GDC: | |
| tl.extra.cuda.gdc_launch_dependents() | |
| def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor, | |
| eps: float) -> torch.Tensor: | |
| output = torch.empty_like(x) | |
| rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device) | |
| block_n = triton.next_power_of_2(x.shape[1]) | |
| pdl = torch.cuda.get_device_capability(x.device)[0] >= 9 | |
| launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {} | |
| _layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)]( | |
| x, output, weight, None, z, None, rstd, | |
| x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps, | |
| BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True, | |
| NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish", | |
| num_warps=min(max(block_n // 256, 1), 8), **launch, | |
| ) | |
| return output | |
| def _fused_sigmoid_mul_kernel( | |
| output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor, | |
| gate_stride_row: tl.int64, gate_stride_head: tl.int64, | |
| hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr, | |
| ) -> None: | |
| """Fuse sigmoid(gate) * attn_output into a single kernel.""" | |
| pid_row = tl.program_id(0).to(tl.int64) | |
| pid_block = tl.program_id(1) | |
| offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H) | |
| mask = offsets < hidden_dim | |
| head = offsets // HEAD_DIM | |
| d = offsets - head * HEAD_DIM | |
| attn_off = pid_row * hidden_dim + offsets | |
| attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32) | |
| gate_off = pid_row * gate_stride_row + head * gate_stride_head + d | |
| g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32) | |
| result = attn * tl.sigmoid(g) | |
| tl.store(output_ptr + attn_off, result, mask=mask) | |
| def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor: | |
| """ | |
| Fused sigmoid-mul for attention output gating. | |
| Equivalent to: attn_output * sigmoid(gate) | |
| The production Qwen3.5 path passes a 3D strided gate. A single hidden-block | |
| Triton kernel handles both that path and flat contiguous inputs. | |
| When inplace=True, writes result back to attn_output and returns it. | |
| Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim) | |
| and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate | |
| via explicit strides without requiring a contiguous copy. | |
| """ | |
| if gate.ndim == 3 and attn_output.ndim == 2: | |
| num_tokens, num_heads, head_dim = gate.shape | |
| hidden_dim = num_heads * head_dim | |
| assert attn_output.shape == (num_tokens, hidden_dim) | |
| gate_stride_row = gate.stride(0) | |
| gate_stride_head = gate.stride(1) | |
| else: | |
| assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape' | |
| hidden_dim = attn_output.shape[-1] | |
| num_tokens = attn_output.numel() // hidden_dim | |
| head_dim = hidden_dim | |
| gate_stride_row = hidden_dim | |
| gate_stride_head = hidden_dim | |
| out = attn_output if inplace else torch.empty_like(attn_output) | |
| block_h = 1024 if num_tokens < 1024 else 2048 | |
| grid = (num_tokens, triton.cdiv(hidden_dim, block_h)) | |
| _fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4) | |
| return out | |
| def _silu_mul_kernel( | |
| gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor, | |
| elements: tl.constexpr, BLOCK: tl.constexpr, | |
| ) -> None: | |
| offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) | |
| mask = offsets < elements | |
| gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32) | |
| up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32) | |
| # Preserve the qualified CUDA --use_fast_math operations, including FTZ. | |
| output = tl.inline_asm_elementwise( | |
| "{ .reg .f32 exponent, denominator, activated; " | |
| "mul.ftz.f32 exponent, $1, 0fBFB8AA3B; " | |
| "ex2.approx.ftz.f32 exponent, exponent; " | |
| "add.ftz.f32 denominator, exponent, 0f3F800000; " | |
| "div.approx.ftz.f32 activated, $1, denominator; " | |
| "mul.ftz.f32 $0, $2, activated; }", | |
| constraints="=f,f,f", args=[gate, up], dtype=tl.float32, | |
| is_pure=True, pack=1, | |
| ) | |
| tl.store(output_ptr + offsets, output, mask=mask) | |
| def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor: | |
| """Match the qualified fused CUDA SiLU-mul before the BF16 output store.""" | |
| if (gate.shape != up.shape or gate.device != up.device | |
| or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16 | |
| or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()): | |
| raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors") | |
| output = torch.empty_like(gate) | |
| _silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)]( | |
| gate, up, output, gate.numel(), BLOCK=2048, num_warps=8, | |
| ) | |
| return output | |
| # MIT License | |
| # | |
| # Copyright 2025 Thinking Machines Lab | |
| # | |
| # Permission is hereby granted, free of charge, to any person obtaining a copy | |
| # of this software and associated documentation files (the "Software"), to deal | |
| # in the Software without restriction, including without limitation the rights | |
| # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |
| # copies of the Software, and to permit persons to whom the Software is | |
| # furnished to do so, subject to the following conditions: | |
| # | |
| # The above copyright notice and this permission notice shall be included in all | |
| # copies or substantial portions of the Software. | |
| # | |
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |
| # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
| # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| # SOFTWARE. | |
| def get_device_core_count() -> int: | |
| return torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count | |
| def _compute_pid(tile_id: tl.tensor, num_pid_in_group: tl.tensor, num_pid_m: tl.tensor, GROUP_SIZE_M: tl.tensor, NUM_SMS: tl.tensor) -> tuple[tl.tensor, tl.tensor]: | |
| group_id = tile_id // num_pid_in_group | |
| first_pid_m = group_id * GROUP_SIZE_M | |
| group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) | |
| pid_m = first_pid_m + tile_id % group_size_m | |
| pid_n = tile_id % num_pid_in_group // group_size_m | |
| return (pid_m, pid_n) | |
| def matmul_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, bias_ptr: tl.tensor, M: tl.int32, N: tl.int32, K: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr, HAS_BIAS: tl.constexpr) -> None: | |
| start_pid = tl.program_id(axis=0) | |
| num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) | |
| num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) | |
| k_tiles = tl.cdiv(K, BLOCK_SIZE_K) | |
| num_tiles = num_pid_m * num_pid_n | |
| offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) | |
| num_pid_in_group = GROUP_SIZE_M * num_pid_n | |
| for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True): | |
| (pid_m, pid_n) = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS) | |
| start_m = pid_m * BLOCK_SIZE_M | |
| start_n = pid_n * BLOCK_SIZE_N | |
| offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) | |
| offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) | |
| if A_LARGE: | |
| offs_am = offs_am.to(tl.int64) | |
| if B_LARGE: | |
| offs_bn = offs_bn.to(tl.int64) | |
| offs_am = tl.where(offs_am < M, offs_am, 0) | |
| offs_bn = tl.where(offs_bn < N, offs_bn, 0) | |
| offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) | |
| offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) | |
| accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) | |
| for ki in range(k_tiles): | |
| if A_LARGE or B_LARGE: | |
| offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) | |
| else: | |
| offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) | |
| a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) | |
| b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) | |
| a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0) | |
| b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0) | |
| accumulator = tl.dot(a, b, accumulator) | |
| offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) | |
| offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) | |
| if C_LARGE: | |
| offs_cm = offs_cm.to(tl.int64) | |
| offs_cn = offs_cn.to(tl.int64) | |
| c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] | |
| c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) | |
| if HAS_BIAS: | |
| bias_ptrs = bias_ptr + offs_cn | |
| bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32) | |
| accumulator += bias | |
| if c_ptr.dtype.element_ty == tl.float8e4nv: | |
| c = accumulator.to(tl.float8e4nv) | |
| elif c_ptr.dtype.element_ty == tl.bfloat16: | |
| c = accumulator.to(tl.bfloat16) | |
| elif c_ptr.dtype.element_ty == tl.float32: | |
| c = accumulator.to(tl.float32) | |
| else: | |
| c = accumulator.to(tl.float16) | |
| tl.store(c_ptrs, c, mask=c_mask) | |
| def _matmul_persistent_triton(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None=None) -> torch.Tensor: | |
| assert a.shape[1] == b.shape[0], 'Incompatible dimensions' | |
| assert a.dtype == b.dtype, 'Incompatible dtypes' | |
| assert bias is None or bias.dim() == 1, 'Expected a one-dimensional bias' | |
| NUM_SMS = get_device_core_count() | |
| (M, K) = a.shape | |
| (K, N) = b.shape | |
| dtype = a.dtype | |
| c = torch.empty((M, N), device=a.device, dtype=dtype) | |
| def grid(META: dict[str, Any]) -> tuple[int]: | |
| return (min(NUM_SMS, triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N'])),) | |
| configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}} | |
| matmul_kernel_persistent[grid](a, b, c, bias, M, N, K, a.stride(0), a.stride(1), b.stride(0), b.stride(1), c.stride(0), c.stride(1), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, HAS_BIAS=bias is not None, **configs[dtype]) | |
| return c | |
| def mean_kernel(input_ptr: tl.tensor, output_ptr: tl.tensor, input_stride0: tl.int32, input_stride1: tl.int32, input_stride2: tl.int32, output_stride0: tl.int32, output_stride1: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, BLOCK_SIZE: tl.constexpr) -> None: | |
| """ | |
| Kernel for computing mean along a single dimension. | |
| Input is viewed as (M, N, K) where N is the dimension being reduced. | |
| """ | |
| pid = tl.program_id(0) | |
| m_idx = pid // K | |
| k_idx = pid % K | |
| if m_idx >= M or k_idx >= K: | |
| return | |
| acc = 0.0 | |
| for n_start in range(0, N, BLOCK_SIZE): | |
| n_offsets = n_start + tl.arange(0, BLOCK_SIZE) | |
| mask = n_offsets < N | |
| input_idx = m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2 | |
| vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0) | |
| acc += tl.sum(vals) | |
| mean_val = acc / N | |
| output_idx = m_idx * output_stride0 + k_idx * output_stride1 | |
| tl.store(output_ptr + output_idx, mean_val) | |
| def mean_dim(input: torch.Tensor, dim: int, keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor: | |
| """ | |
| Triton implementation of torch.mean with single dimension reduction. | |
| Args: | |
| input: Input tensor | |
| dim: Single dimension along which to compute mean | |
| keepdim: Whether to keep the reduced dimension | |
| dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs) | |
| Returns: | |
| Tensor with mean values along specified dimension | |
| """ | |
| assert input.is_cuda or input.is_xpu, 'Input must be a CUDA or XPU tensor' | |
| assert -input.ndim <= dim < input.ndim, f'Invalid dimension {dim} for tensor with {input.ndim} dimensions' | |
| if dim < 0: | |
| dim = dim + input.ndim | |
| if dtype is None: | |
| if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]: | |
| dtype = torch.float32 | |
| else: | |
| dtype = input.dtype | |
| if input.dtype != dtype: | |
| input = input.to(dtype) | |
| shape = list(input.shape) | |
| M = 1 | |
| for i in range(dim): | |
| M *= shape[i] | |
| N = shape[dim] | |
| K = 1 | |
| for i in range(dim + 1, len(shape)): | |
| K *= shape[i] | |
| input_3d = input.reshape(M, N, K) | |
| if keepdim: | |
| output_shape = shape.copy() | |
| output_shape[dim] = 1 | |
| else: | |
| output_shape = shape[:dim] + shape[dim + 1:] | |
| output = torch.empty(output_shape, dtype=dtype, device=input.device) | |
| if keepdim: | |
| output_2d = output.reshape(M, 1, K).squeeze(1) | |
| else: | |
| output_2d = output.reshape(M, K) | |
| grid = (M * K,) | |
| BLOCK_SIZE = 1024 | |
| mean_kernel[grid](input_3d, output_2d, input_3d.stride(0), input_3d.stride(1), input_3d.stride(2), output_2d.stride(0), output_2d.stride(1) if output_2d.ndim > 1 else 0, M, N, K, BLOCK_SIZE) | |
| return output | |
| def mean_batch_invariant(input: torch.Tensor, dim: list[int], keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor: | |
| assert dtype is None or dtype == torch.float32, f'unsupported dtype: {dtype}' | |
| if len(dim) == 1: | |
| return mean_dim(input, dim[0], keepdim=keepdim) | |
| else: | |
| assert input.dtype in {torch.float16, torch.bfloat16, torch.float32}, 'only float types supported for now' | |
| n_elems = 1 | |
| for d in dim: | |
| n_elems *= input.shape[d] | |
| return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems | |
| def bmm_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, B: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, stride_ab: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bb: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cb: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr) -> None: | |
| """ | |
| Batched matrix multiplication kernel that processes batches in parallel. | |
| Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch. | |
| """ | |
| start_pid = tl.program_id(axis=0) | |
| num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) | |
| num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) | |
| k_tiles = tl.cdiv(K, BLOCK_SIZE_K) | |
| num_tiles_per_batch = num_pid_m * num_pid_n | |
| num_tiles_total = B * num_tiles_per_batch | |
| offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) | |
| num_pid_in_group = GROUP_SIZE_M * num_pid_n | |
| for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True): | |
| batch_idx = tile_id // num_tiles_per_batch | |
| tile_in_batch = tile_id % num_tiles_per_batch | |
| (pid_m, pid_n) = _compute_pid(tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS) | |
| start_m = pid_m * BLOCK_SIZE_M | |
| start_n = pid_n * BLOCK_SIZE_N | |
| offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) | |
| offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) | |
| if A_LARGE: | |
| offs_am = offs_am.to(tl.int64) | |
| if B_LARGE: | |
| offs_bn = offs_bn.to(tl.int64) | |
| offs_am = tl.where(offs_am < M, offs_am, 0) | |
| offs_bn = tl.where(offs_bn < N, offs_bn, 0) | |
| offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) | |
| offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) | |
| if A_LARGE or B_LARGE: | |
| batch_idx_typed = batch_idx.to(tl.int64) | |
| else: | |
| batch_idx_typed = batch_idx | |
| accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) | |
| for ki in range(k_tiles): | |
| if A_LARGE or B_LARGE: | |
| offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) | |
| else: | |
| offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) | |
| a_ptrs = a_ptr + (batch_idx_typed * stride_ab + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) | |
| b_ptrs = b_ptr + (batch_idx_typed * stride_bb + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) | |
| a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0) | |
| b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0) | |
| accumulator = tl.dot(a, b, accumulator) | |
| offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) | |
| offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) | |
| if C_LARGE: | |
| offs_cm = offs_cm.to(tl.int64) | |
| offs_cn = offs_cn.to(tl.int64) | |
| c_ptrs = c_ptr + batch_idx_typed * stride_cb + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] | |
| c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) | |
| if c_ptr.dtype.element_ty == tl.float8e4nv: | |
| c = accumulator.to(tl.float8e4nv) | |
| elif c_ptr.dtype.element_ty == tl.bfloat16: | |
| c = accumulator.to(tl.bfloat16) | |
| elif c_ptr.dtype.element_ty == tl.float32: | |
| c = accumulator.to(tl.float32) | |
| else: | |
| c = accumulator.to(tl.float16) | |
| tl.store(c_ptrs, c, mask=c_mask) | |
| def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor | None=None) -> torch.Tensor: | |
| if a.ndim == 3 and b.ndim == 3: | |
| assert a.shape[0] == b.shape[0], 'Batch sizes must match' | |
| assert a.shape[2] == b.shape[1], 'Incompatible dimensions' | |
| assert a.dtype == b.dtype, 'Incompatible dtypes' | |
| B = a.shape[0] | |
| M = a.shape[1] | |
| K = a.shape[2] | |
| N = b.shape[2] | |
| dtype = a.dtype | |
| if out is None: | |
| c = torch.empty((B, M, N), device=a.device, dtype=dtype) | |
| else: | |
| c = out | |
| NUM_SMS = get_device_core_count() | |
| configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}} | |
| config = configs.get(dtype) | |
| if config is None: | |
| raise ValueError(f'Unsupported dtype {dtype} for bmm_batch_invariant. Supported dtypes are: {list(configs.keys())}') | |
| num_tiles_per_batch = triton.cdiv(M, config['BLOCK_SIZE_M']) * triton.cdiv(N, config['BLOCK_SIZE_N']) | |
| num_tiles_total = B * num_tiles_per_batch | |
| grid = (min(NUM_SMS, num_tiles_total),) | |
| bmm_kernel_persistent[grid](a, b, c, B, M, N, K, a.stride(0), a.stride(1), a.stride(2), b.stride(0), b.stride(1), b.stride(2), c.stride(0), c.stride(1), c.stride(2), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, **config) | |
| return c | |
| else: | |
| raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}') | |
| def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor, | |
| count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None: | |
| offsets = tl.program_id(0) * block + tl.arange(0, block) | |
| current = tl.load(value + offsets, offsets < count, other=0) | |
| clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL), | |
| limit, propagate_nan=tl.PropagateNan.ALL) | |
| upper = clipped.to(tl.bfloat16) | |
| lower = (current - upper.to(tl.float32)).to(tl.bfloat16) | |
| lower = tl.where(current == 0, upper, lower) | |
| tl.store(high + offsets, upper, offsets < count) | |
| tl.store(low + offsets, lower, offsets < count) | |
| def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous(): | |
| raise ValueError("Expected contiguous CUDA FP32 residuals") | |
| high = torch.empty_like(value, dtype=torch.bfloat16) | |
| low = torch.empty_like(value, dtype=torch.bfloat16) | |
| count = value.numel() | |
| if count: | |
| _bf16_pair_kernel[(triton.cdiv(count, 1024),)]( | |
| value, high, low, count, torch.finfo(torch.bfloat16).max, 1024, | |
| num_warps=4, enable_fp_fusion=False, | |
| ) | |
| return high, low | |
| def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor, | |
| count: tl.int32, block: tl.constexpr) -> None: | |
| offsets = tl.program_id(0) * block + tl.arange(0, block) | |
| upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32) | |
| lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32) | |
| tl.store(output + offsets, upper + lower, offsets < count) | |
| def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor: | |
| if (not first.is_cuda or second.device != first.device or second.shape != first.shape | |
| or first.dtype not in (torch.bfloat16, torch.float32) | |
| or second.dtype != torch.bfloat16 | |
| or not first.is_contiguous() or not second.is_contiguous()): | |
| raise ValueError("Expected aligned contiguous CUDA residuals") | |
| output = torch.empty_like(first, dtype=torch.float32) | |
| count = first.numel() | |
| if count: | |
| _residual_kernel[(triton.cdiv(count, 1024),)]( | |
| first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False, | |
| ) | |
| return output | |