d3-FP8 / d3q /kernels.py
subin's picture
Index FP8 activation rows in 64 bits and add Decision Index results
5887468 verified
Raw History Blame Contribute Delete
9.78 kB
"""Linear layers for d3's quantized checkpoints (compressed-tensors), and the loader that installs them.
transformers loads compressed-tensors weights without the d3 runtime's assumptions holding (it reads BF16
``Linear`` weights), so ``load_backbone`` loads the checkpoint compressed and replaces every quantized layer:
- ``Int4Linear``: ``pack-quantized`` int4 weights in groups, symmetric (GPTQ) or with zero points (AWQ), kept
packed on the GPU. Each call unpacks the weight to BF16 with one Triton kernel and multiplies in BF16, which
computes exactly what a BF16 layer holding the dequantized weight computes. It saves memory, not time.
- ``Float8Linear``: ``float-quantized`` FP8 weights with one scale per output channel (FP8_DYNAMIC). Each call
quantizes the activations to FP8 per row with one Triton kernel and multiplies in ``torch._scaled_mm``. It
needs FP8 matrix units: AMD Instinct MI300 / MI350 series, or NVIDIA compute capability 8.9 or later. On MI300
(gfx942), whose units multiply ``float8_e4m3fnuz``, the stored E4M3 bits are reinterpreted and the scales
doubled, which keeps every value. On Ada (8.9) the per-row scaled product takes a slow kernel from 256 rows
on, so there the product is taken unscaled in FP32 and the scales applied after.
These are the layers of the vLLM Semantic Router model runtime (``engines/native/quantized.py``).
"""
from __future__ import annotations
from typing import Any
import torch
import triton
import triton.language as tl
from torch import nn
PACK = 8
UNPACK_BLOCK = 256
ROCM_FLOAT8 = {"gfx942": torch.float8_e4m3fnuz, "gfx950": torch.float8_e4m3fn}
@triton.jit
def _unpack_int4(packed_ptr, scale_ptr, zero_ptr, out_ptr, n_words, words_per_row, groups_per_row,
GROUP: tl.constexpr, BLOCK: tl.constexpr, ZERO: tl.constexpr):
words = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = words < n_words
word = tl.load(packed_ptr + words, mask=mask, other=0)
row = words // words_per_row
column = (words % words_per_row) * 8
scale = tl.load(scale_ptr + row * groups_per_row + column // GROUP, mask=mask, other=0.0).to(tl.float32)
nibbles = (word[:, None] >> (tl.arange(0, 8) * 4)[None, :]) & 0xF
values = (nibbles - 8).to(tl.float32)
if ZERO:
zero_word = tl.load(zero_ptr + (row // 8) * groups_per_row + column // GROUP, mask=mask, other=0)
values -= (((zero_word >> ((row % 8) * 4)) & 0xF) - 8).to(tl.float32)[:, None]
values = values * scale[:, None]
out = words[:, None] * 8 + tl.arange(0, 8)[None, :]
tl.store(out_ptr + out, values.to(out_ptr.dtype.element_ty), mask=mask[:, None])
@triton.jit
def _quantize_rows(x_ptr, q_ptr, s_ptr, n_cols, stride_x, stride_q, fmax, BLOCK: tl.constexpr):
row = tl.program_id(0).to(tl.int64) # rows * stride passes 2**31 on long prompts
x_row = x_ptr + row * stride_x
amax = tl.zeros((BLOCK,), dtype=tl.float32)
for start in range(0, n_cols, BLOCK):
cols = start + tl.arange(0, BLOCK)
amax = tl.maximum(amax, tl.abs(tl.load(x_row + cols, mask=cols < n_cols, other=0.0).to(tl.float32)))
scale = tl.maximum(tl.max(amax, axis=0), 1e-12) / fmax
tl.store(s_ptr + row, scale)
inv = 1.0 / scale
q_row = q_ptr + row * stride_q
for start in range(0, n_cols, BLOCK):
cols = start + tl.arange(0, BLOCK)
x = tl.load(x_row + cols, mask=cols < n_cols, other=0.0).to(tl.float32)
q = tl.minimum(tl.maximum(x * inv, -fmax), fmax)
tl.store(q_row + cols, q.to(q_ptr.dtype.element_ty), mask=cols < n_cols)
def float8_format(device: torch.device) -> torch.dtype | None:
if device.type != "cuda":
return None
if torch.version.hip is not None:
return ROCM_FLOAT8.get(torch.cuda.get_device_properties(device).gcnArchName.split(":")[0])
return torch.float8_e4m3fn if torch.cuda.get_device_capability(device) >= (8, 9) else None
class Int4Linear(nn.Module):
def __init__(self, packed, scale, zero, in_features, out_features, bias=None):
super().__init__()
self.in_features, self.out_features = in_features, out_features
self.group_size = in_features // scale.shape[1]
self.register_buffer("weight_packed", packed.to(torch.int32).contiguous())
self.register_buffer("weight_scale", scale.contiguous())
self.register_buffer("weight_zero_point", None if zero is None else zero.to(torch.int32).contiguous())
self.bias = bias
def unpacked(self) -> torch.Tensor:
rows, words = self.weight_packed.shape
out = torch.empty(rows, words * PACK, dtype=torch.bfloat16, device=self.weight_packed.device)
n = self.weight_packed.numel()
zero = self.weight_zero_point
_unpack_int4[(triton.cdiv(n, UNPACK_BLOCK),)](
self.weight_packed, self.weight_scale, self.weight_packed if zero is None else zero, out, n, words,
self.weight_scale.shape[1], GROUP=self.group_size, BLOCK=UNPACK_BLOCK, ZERO=zero is not None)
return out
def forward(self, x: torch.Tensor) -> torch.Tensor:
return nn.functional.linear(x, self.unpacked().to(x.dtype), self.bias)
def extra_repr(self) -> str:
return f"in={self.in_features}, out={self.out_features}, int4 g{self.group_size}"
class Float8Linear(nn.Module):
def __init__(self, weight, scale, in_features, out_features, bias=None):
super().__init__()
fmt = float8_format(weight.device)
if fmt is None:
raise RuntimeError("FP8 weights need an AMD Instinct MI300 / MI350 or an NVIDIA GPU of compute "
"capability 8.9 or later")
scale = scale.float().reshape(out_features, 1)
if fmt == torch.float8_e4m3fnuz and weight.dtype == torch.float8_e4m3fn:
bits = weight.view(torch.int8)
weight = torch.where(bits == -128, torch.zeros_like(bits), bits).view(torch.float8_e4m3fnuz)
scale = scale * 2 # the same bits mean half the value in e4m3fnuz
self.in_features, self.out_features = in_features, out_features
self.register_buffer("weight", weight.contiguous())
self.register_buffer("weight_scale", scale.t().contiguous())
self.unscaled = torch.version.hip is None and torch.cuda.get_device_capability(weight.device) == (8, 9)
self.register_buffer("one", torch.ones((), device=weight.device))
self.bias = bias
def forward(self, x: torch.Tensor) -> torch.Tensor:
shape = x.shape
rows = x.reshape(-1, shape[-1]).contiguous()
q = torch.empty(rows.shape, device=rows.device, dtype=self.weight.dtype)
s = torch.empty(rows.shape[0], 1, device=rows.device, dtype=torch.float32)
if rows.shape[0]:
block = min(triton.next_power_of_2(rows.shape[1]), 4096)
_quantize_rows[(rows.shape[0],)](rows, q, s, rows.shape[1], rows.stride(0), q.stride(0),
torch.finfo(self.weight.dtype).max, BLOCK=block, num_warps=8)
if self.unscaled:
out = torch._scaled_mm(q, self.weight.t(), scale_a=self.one, scale_b=self.one, out_dtype=torch.float32)
out = (out * s * self.weight_scale).to(x.dtype)
else:
out = torch._scaled_mm(q, self.weight.t(), scale_a=s, scale_b=self.weight_scale, out_dtype=x.dtype)
if self.bias is not None:
out = out + self.bias
return out.reshape(*shape[:-1], self.out_features)
def extra_repr(self) -> str:
return f"in={self.in_features}, out={self.out_features}, fp8"
def _tensors(module: nn.Module) -> dict[str, torch.Tensor]:
return {name: t for name, t in (*module.named_parameters(recurse=False), *module.named_buffers(recurse=False))
if t is not None}
def convert(model: nn.Module) -> dict[str, int]:
"""Replace every compressed module of a ``dequantize=False`` load with its kernel layer."""
counts = {"Int4Linear": 0, "Float8Linear": 0}
for name, module in list(model.named_modules()):
t = _tensors(module)
if "weight_packed" in t:
shape = t.get("weight_shape")
out_features, in_features = (int(shape[0]), int(shape[1])) if shape is not None else (
t["weight_packed"].shape[0], t["weight_packed"].shape[1] * PACK)
new = Int4Linear(t["weight_packed"].data, t["weight_scale"].data, t.get("weight_zero_point"),
in_features, out_features, t.get("bias"))
elif "weight" in t and t["weight"].dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz) and "weight_scale" in t:
out_features, in_features = t["weight"].shape
new = Float8Linear(t["weight"].data, t["weight_scale"].data, in_features, out_features, t.get("bias"))
else:
continue
parent, _, child = name.rpartition(".")
setattr(model.get_submodule(parent) if parent else model, child, new)
counts[type(new).__name__] += 1
return counts
def load_backbone(path: str, device: str = "cuda:0", **kwargs: Any):
"""A quantized ``Qwen3_5Model`` with its weights kept quantized and kernel layers installed."""
from transformers import CompressedTensorsConfig, Qwen3_5Model
kwargs.pop("quantization_config", None)
kwargs.setdefault("dtype", torch.bfloat16)
kwargs.setdefault("device_map", {"": device})
model = Qwen3_5Model.from_pretrained(path, quantization_config=CompressedTensorsConfig(dequantize=False),
**kwargs)
model.quantized_modules = convert(model)
if not any(model.quantized_modules.values()):
raise RuntimeError("no quantized layer found: the checkpoint did not load compressed")
return model