clairic / clairic_runtime /normalization.py
areneau's picture
Update Clairic Transformers runtime
524fd9d verified
Raw History Blame Contribute Delete
1.44 kB
"""Public CUDA normalization with the model's residual precision."""
from __future__ import annotations
import torch
def _bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Recover BF16 operands for an FP32 sum of two BF16 residuals."""
if (value.is_cuda and value.dtype == torch.float32 and value.is_contiguous()
and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
from .kernels import bf16_pair_sum
return bf16_pair_sum(value)
limit = torch.finfo(torch.bfloat16).max
high = value.clamp(-limit, limit).to(torch.bfloat16)
low = (value - high.float()).to(torch.bfloat16)
# Equal signed zeros keep the sign when the fused operation adds them.
low = torch.where(value == 0, high, low)
return high, low
def rms_norm(value: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
"""Normalize CUDA BF16 states or unrounded FP32 residual sums."""
from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
shape = value.shape
flat = value.reshape(-1, shape[-1]).contiguous()
if value.dtype == torch.bfloat16:
return gemma_rmsnorm(flat, weight, eps).reshape(shape)
if value.dtype != torch.float32:
raise ValueError("Expected BF16 states or FP32 residual sums")
high, low = _bf16_pair_sum(flat)
gemma_fused_add_rmsnorm(low, high, weight, eps)
return low.reshape(shape)