# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/modules/l2norm.py # -*- coding: utf-8 -*- # Copyright (c) 2023-2025, Songlin Yang, Yu Zhang # Standalone import layout; see LICENSES and NOTICE for retained notices. from typing import Optional import torch import triton import triton.language as tl @triton.jit def l2norm_fwd_kernel1( x: tl.tensor, y: tl.tensor, D: tl.int32, BD: tl.constexpr, eps: tl.float32, ) -> None: i_t = tl.program_id(0) x += i_t * D y += i_t * D # Compute mean and variance cols = tl.arange(0, BD) mask = cols < D b_x = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32) b_var = tl.sum(b_x * b_x, axis=0) b_rstd = 1 / tl.sqrt(b_var + eps) # tl.store(Rstd + i_t, rstd) # Normalize and apply linear transformation b_y = b_x * b_rstd tl.store(y + cols, b_y, mask=mask) @triton.jit(do_not_specialize=["T"]) def l2norm_fwd_kernel( x: tl.tensor, y: tl.tensor, eps: tl.float32, T: tl.int32, D: tl.constexpr, BT: tl.constexpr, BD: tl.constexpr, ) -> None: i_t = tl.program_id(0) p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32) b_var = tl.sum(b_x * b_x, axis=1) b_y = b_x / tl.sqrt(b_var + eps)[:, None] p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1)) def l2norm_fwd( x: torch.Tensor, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None ) -> torch.Tensor: x_shape_og = x.shape x = x.view(-1, x.shape[-1]) # allocate output if output_dtype is None: y = torch.empty_like(x) else: y = torch.empty_like(x, dtype=output_dtype) assert y.stride(-1) == 1 T, D = x.shape[0], x.shape[-1] # rstd = torch.empty((T,), dtype=torch.float32, device=x.device) # Less than 64KB per feature: enqueue fused kernel MAX_FUSED_SIZE = 65536 // x.element_size() BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) if D > BD: raise RuntimeError("This layer doesn't support feature dim >= 64KB.") if D <= 512: def grid(meta: dict[str, int]) -> tuple[int]: return (triton.cdiv(T, meta["BT"]),) l2norm_fwd_kernel[grid]( x, y, eps, T=T, D=D, BD=BD, BT=16, num_warps=8, num_stages=3, ) else: l2norm_fwd_kernel1[(T,)]( x, y, eps=eps, D=D, BD=BD, num_warps=8, num_stages=3, ) return y.view(x_shape_og)