Blame for build/torch-rocm/moe/quant_moe.py

from enum import Enum
import triton
import torch
from .._triton_kernels.moe.quant_moe import (
    _downcast_to_static_fp8,
    _downcast_to_mxfp,
    _upcast_from_mxfp,
    _smoothquant_fuse_quant_kernel,
    _smoothquant_fuse_quant_kernel_single_pass,
)
from ..utils._triton.arch_info import get_arch


def downcast_to_static_fp8_3d(x: torch.Tensor, scale: torch.Tensor):
    assert x.ndim == 3
    E, M, N = x.shape

    x2d = x.reshape(E * M, N).contiguous()

    y2d = downcast_to_static_fp8(x2d, scale)
    y3d = y2d.reshape(E, M, N)
    return y3d


def downcast_to_static_fp8(x: torch.Tensor, scale: torch.Tensor):
    M, N = x.shape
    if get_arch() != "gfx942":
        dtype = torch.float8_e4m3fn
    else:
        dtype = torch.float8_e4m3fnuz
    y = torch.empty((M, N), dtype=dtype, device="cuda")

    BLOCK_M = min(triton.next_power_of_2(M), 128)
    if M <= 4096:
        BLOCK_N = 32
    else:
        BLOCK_N = 64
    grid_m = triton.cdiv(x.shape[0], BLOCK_M)
    grid_n = triton.cdiv(x.shape[1], BLOCK_N)

    _downcast_to_static_fp8[(grid_m, grid_n)](
        x,
        x.stride(0),
        x.stride(1),
        y,
        y.stride(0),
        y.stride(1),
        scale,
        M,
        N,
        BLOCK_M,
        BLOCK_N,
        num_warps=8,
    )

    return y


class DequantScaleRoundingMode(Enum):
    ROUND_UP = 0
    ROUND_DOWN = 1


def downcast_to_mxfp(
    src_tensor: torch.Tensor,
    out_quant_type: torch.dtype,
    axis: int,
    DEQUANT_SCALE_ROUNDING_MODE: DequantScaleRoundingMode = DequantScaleRoundingMode.ROUND_UP,
):
    """
    Convert the src weights to mx format. The src weight is quantized along the axis dimension.

    If weight_quant_type is torch.uint8, we output mxfp4 where two e2m1 values are packed into a single byte.
    Note that this means the k_dim of the tensor will be half of the logical k_dim.

    If weight_quant_type is torch.float8_e4m3fn or torch.float8_e5m2, we output mxfp8 with the float8s are stored
    in their respective formats.
    """
    ndim = src_tensor.ndim
    assert -ndim <= axis < ndim, f"Invalid axis {axis=}"
    axis = axis if axis >= 0 else axis + ndim
    # downcast
    src_tensor = src_tensor.transpose(axis, src_tensor.ndim - 1)
    is_fp4 = out_quant_type == torch.uint8
    is_fp8 = out_quant_type in (
        torch.float8_e4m3fn,
        torch.float8_e4m3fnuz,
        torch.float8_e5m2,
    )
    assert is_fp4 or is_fp8
    divisor = 2 if is_fp4 else 1
    L = src_tensor.shape[-1]
    if is_fp4:
        assert L % 2 == 0, f"axis dim must be divisible by 2 for e2m1. Got {L}"
    out_shape = src_tensor.shape[:-1] + (L // divisor,)
    out_scale_shape = src_tensor.shape[:-1] + (triton.cdiv(L, 32),)

    out_quant_tensor = src_tensor.new_empty(out_shape, dtype=out_quant_type)
    out_scale = src_tensor.new_empty(out_scale_shape, dtype=torch.uint8)

    kernel_src_tensor = src_tensor.reshape(-1, src_tensor.shape[-1])
    kernel_quant_tensor = out_quant_tensor.view(-1, out_quant_tensor.shape[-1])
    kernel_scale = out_scale.view(-1, out_scale.shape[-1])

    BLOCK_OUT_DIM = 128
    BLOCK_QUANT_DIM = 32
    grid_out = triton.cdiv(kernel_src_tensor.shape[0], BLOCK_OUT_DIM)
    grid_quant = triton.cdiv(kernel_src_tensor.shape[1], BLOCK_QUANT_DIM)

    _downcast_to_mxfp[(grid_out, grid_quant)](
        kernel_quant_tensor,
        *kernel_quant_tensor.stride(),
        kernel_scale,
        *kernel_scale.stride(),
        kernel_src_tensor,
        *kernel_src_tensor.stride(),
        *kernel_src_tensor.shape,
        BLOCK_OUT_DIM,
        BLOCK_QUANT_DIM,
        DEQUANT_SCALE_ROUNDING_MODE.value,
        num_warps=8,
    )

    out_quant_tensor = out_quant_tensor.transpose(axis, src_tensor.ndim - 1)
    out_scale = out_scale.transpose(axis, src_tensor.ndim - 1)
    return out_quant_tensor, out_scale


def upcast_from_mxfp(
    tensor: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, axis: int
):
    """
    Upcasts an mxfp (packed) weight tensor back to float16 or bfloat16.

    The function assumes that the tensors were quantized along the given axis.
    It permutes the tensor so that the quantized axis is last, reshapes to 2D,
    launches the Triton upcast kernel, and then unpermutes back to the original order.
    """
    ndim = tensor.ndim
    assert -ndim <= axis < ndim, f"Invalid axis {axis=}"
    axis = axis if axis >= 0 else axis + ndim
    assert tensor.ndim == scale.ndim, (
        f"Weight and scale must have the same number of dimensions. "
        f"Got {tensor.ndim=} and {scale.ndim=}"
    )
    # dtype checks
    assert tensor.dtype in {
        torch.uint8,
        torch.float8_e5m2,
        torch.float8_e4m3fn,
        torch.float8_e4m3fnuz,
    }, f"Invalid tensor dtype {tensor.dtype=}"
    assert scale.dtype == torch.uint8, f"Invalid scale dtype {scale.dtype=}"
    assert dtype in (torch.float16, torch.bfloat16), f"Invalid output dtype {dtype=}"
    # upcast
    logical_quant_dim = tensor.shape[axis] * (2 if tensor.dtype == torch.uint8 else 1)
    tensor = tensor.transpose(axis, tensor.ndim - 1).contiguous()
    scale = scale.transpose(axis, scale.ndim - 1).contiguous()
    out = torch.empty(
        (*tensor.shape[:-1], logical_quant_dim), dtype=dtype, device=tensor.device
    )
    reshaped_out = out.view(-1, out.shape[-1])
    reshaped_tensor = tensor.view(-1, tensor.shape[-1])
    reshaped_scale = scale.view(-1, scale.shape[-1])
    BLOCK_OUT_DIM = 128
    BLOCK_QUANT_DIM = 32
    blocks_out_dim = triton.cdiv(reshaped_out.shape[0], BLOCK_OUT_DIM)
    blocks_quant_dim = triton.cdiv(reshaped_out.shape[1], BLOCK_QUANT_DIM)
    _upcast_from_mxfp[(blocks_out_dim, blocks_quant_dim)](
        reshaped_out,
        *reshaped_out.stride(),
        reshaped_scale,
        *reshaped_scale.stride(),
        reshaped_tensor,
        *reshaped_tensor.stride(),
        *reshaped_out.shape,
        BLOCK_OUT_DIM,
        BLOCK_QUANT_DIM,
        num_warps=8,
    )
    out = out.transpose(axis, scale.ndim - 1).contiguous()
    return out


def dequant_x_blockscale(x, x_scales, per_row_x_scale, group_shape):
    assert x_scales is not None
    group_shape_m, _, group_shape_k = group_shape
    M, K = x.shape

    K_blocks = (K + group_shape_k - 1) // group_shape_k
    if per_row_x_scale:
        assert x_scales.shape == (M, K_blocks)
        K_pad = K_blocks * group_shape_k
        if K_pad != K:
            x_pad = x.new_zeros((M, K_pad))
            x_pad[:, :K] = x
            x = x_pad

        x = x.to(torch.float32).view(M, K_blocks, group_shape_k) * x_scales.to(
            torch.float32
        ).view(M, K_blocks, 1)
        x = x.view(M, K_pad)[:, :K]
    else:
        M_blocks = (M + group_shape_m - 1) // group_shape_m
        assert x_scales.shape == (M_blocks, K_blocks)
        M_pad = M_blocks * group_shape_m
        K_pad = K_blocks * group_shape_k
        if M_pad != M or K_pad != K:
            x_pad = x.new_zeros((M_pad, K_pad))
            x_pad[:M, :K] = x
            x = x_pad

        x = x.to(torch.float32).view(M_blocks, group_shape_m, K_blocks, group_shape_k)
        scales = x_scales.to(torch.float32).view(M_blocks, 1, K_blocks, 1)
        x = x * scales
        x = x.view(M_pad, K_pad)[:M, :K]
    return x


def dequant_w_blockscale(w, w_scales, group_shape):
    assert w_scales is not None
    _, group_shape_n, group_shape_k = group_shape
    E, K, N = w.shape

    K_blocks = (K + group_shape_k - 1) // group_shape_k
    N_blocks = (N + group_shape_n - 1) // group_shape_n

    assert w_scales.shape == (E, K_blocks, N_blocks)

    K_pad = K_blocks * group_shape_k
    N_pad = N_blocks * group_shape_n
    if K_pad != K or N_pad != N:
        w_pad = w.new_zeros((E, K_pad, N_pad))
        w_pad[:, :K, :N] = w
        w = w_pad
    w = w.to(torch.float32).view(E, K_blocks, group_shape_k, N_blocks, group_shape_n)
    scales = w_scales.to(torch.float32).view(E, K_blocks, 1, N_blocks, 1)
    w = w * scales
    w = w.view(E, K_pad, N_pad)[:, :K, :N]
    return w


def smoothquant_quantize(
    x: torch.Tensor,
    smooth_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    """
    Apply smoothquant quantization to convert bf16/fp16 tensor to int8.

    Args:
        x: Input tensor in bf16/fp16 [M, K]
        smooth_scale: Per-column smooth scale in fp32 [K]

    Returns:
        x_int8: Quantized int8 tensor [M, K]
        x_scale: Per-row quantization scale in fp32 [M]

    The operation performs:
    1. x_smooth = x * smooth_scale (per column)
    2. row_scale = max(abs(x_smooth), dim=1) / 127
    3. x_int8 = round(x_smooth / row_scale)
    """
    assert x.ndim == 2, f"Expected 2D tensor, got {x.ndim}D"
    assert smooth_scale.ndim == 1, f"Expected 1D smooth_scale, got {smooth_scale.ndim}D"
    assert (
        x.shape[1] == smooth_scale.shape[0]
    ), f"Dimension mismatch: x.shape[1]={x.shape[1]}, smooth_scale.shape[0]={smooth_scale.shape[0]}"

    M, K = x.shape
    device = x.device

    x_int8 = torch.empty((M, K), dtype=torch.int8, device=device)
    x_scale = torch.empty((M,), dtype=torch.float32, device=device)

    smooth_scale = smooth_scale.to(torch.float32).contiguous()

    MAX_SINGLE_PASS_K = 1024
    BLOCK_M = min(triton.next_power_of_2(M), 32)

    if K <= MAX_SINGLE_PASS_K:
        # Single pass: load entire row at once
        BLOCK_K = triton.next_power_of_2(K)
        grid = (triton.cdiv(M, BLOCK_M),)

        _smoothquant_fuse_quant_kernel_single_pass[grid](
            x,
            x.stride(0),
            x.stride(1),
            smooth_scale,
            x_int8,
            x_int8.stride(0),
            x_int8.stride(1),
            x_scale,
            1,
            M,
            K,
            BLOCK_M,
            BLOCK_K,
            num_warps=4,
        )
    else:
        BLOCK_K = 256
        grid = (triton.cdiv(M, BLOCK_M),)
        _smoothquant_fuse_quant_kernel[grid](
            x,
            x.stride(0),
            x.stride(1),
            smooth_scale,
            x_int8,
            x_int8.stride(0),
            x_int8.stride(1),
            x_scale,
            1,
            M,
            K,
            BLOCK_M,
            BLOCK_K,
            num_warps=4,
        )

    return x_int8, x_scale


def quantize_weights_int8(
    w: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    """
    Quantize weights to int8 with per-output-channel scaling.

    Args:
        w: Weight tensor in bf16/fp16/fp32 [E, K, N] or [K, N]

    Returns:
        w_int8: Quantized int8 weights (contiguous)
        w_scale: Per-output-channel scale [E, N] or [N] (contiguous)
    """
    if w.ndim == 2:
        # [K, N] -> [1, K, N]
        w = w.unsqueeze(0)
        squeeze_output = True
    else:
        squeeze_output = False

    w_fp32 = w.to(torch.float32)
    w_abs_max = w_fp32.abs().max(dim=1).values
    INT8_MAX = 127.0
    w_scale = w_abs_max / INT8_MAX + 1e-12
    w_scaled = w_fp32 / w_scale[:, None, :]
    w_int8 = w_scaled.round().clamp(-127, 127).to(torch.int8)

    # Layout [E, K, N] with N contiguous
    w_int8 = w_int8.contiguous()
    w_scale = w_scale.contiguous()

    if squeeze_output:
        w_int8 = w_int8.squeeze(0)
        w_scale = w_scale.squeeze(0)

    return w_int8, w_scale