ycchen's picture
Add files using upload-large-folder tool
2925c01 verified
Raw History Blame Contribute Delete
6.77 kB
# Copyright 2026 proof-pilot. Apache-2.0.
"""In-kernel attention sink on patched FA3 (targets >95% of no-sink FA3 for ALL packing).
The FA3 forward kernel epilogue is patched (see flash-attention/hopper/softmax.h finalize)
to add exp(sink-M) to the softmax denominator, so it directly outputs sink-corrected `o`
and sink-inclusive `lse` — NO forward post-pass. Backward feeds those to FA3's native
backward for exact dq/dk/dv (proven equivalent), plus a cheap dsink reduction.
torch.compile: forward and backward are each a `torch.library.custom_op` with a registered
fake (meta) impl, so Dynamo treats them as single opaque nodes with known output shapes
instead of graph-breaking on the fake-less `flash_attn_3.fwd`. The FA3/Triton internals
live in the eager op bodies and are never traced. This lets a compiled decoder layer fuse
everything around attention into one graph (critical for compile + gradient checkpointing,
which re-runs the forward in backward and would otherwise pay the graph-break cost twice).
"""
from __future__ import annotations
import torch
import triton
import triton.language as tl
from flash_attn_interface import _flash_attn_backward
@triton.jit
def _dsink_kernel(o_ptr, do_ptr, sink_ptr, lse2_ptr, dsink_ptr, T, D,
so_t, so_h, sl_h, sl_t, BLOCK_T: tl.constexpr, BLOCK_D: tl.constexpr):
h = tl.program_id(0)
t = tl.program_id(1) * BLOCK_T + tl.arange(0, BLOCK_T)
d = tl.arange(0, BLOCK_D)
tm = t < T
dm = d < D
base = t[:, None] * so_t + h * so_h + d[None, :]
m = tm[:, None] & dm[None, :]
o = tl.load(o_ptr + base, mask=m, other=0.0).to(tl.float32)
g = tl.load(do_ptr + base, mask=m, other=0.0).to(tl.float32)
delta = tl.sum(o * g, axis=1) # [BLOCK_T]
lse2 = tl.load(lse2_ptr + h * sl_h + t * sl_t, mask=tm, other=0.0)
sink = tl.load(sink_ptr + h)
contrib = tl.where(tm, -tl.exp(sink - lse2) * delta, 0.0)
tl.atomic_add(dsink_ptr + h, tl.sum(contrib))
def _dsink(o_sink, do, sink, lse2):
"""dsink_h = -Σ_t exp(sink_h - lse2_{h,t})·(o_sink·do)_{t,h}. o_sink,do [T,H,D]; lse2 [H,T]."""
T, H, D = o_sink.shape
dsink = torch.zeros(H, device=o_sink.device, dtype=torch.float32)
BLOCK_T = 64
BLOCK_D = triton.next_power_of_2(D)
grid = (H, triton.cdiv(T, BLOCK_T))
_dsink_kernel[grid](o_sink, do, sink, lse2, dsink, T, D,
o_sink.stride(0), o_sink.stride(1), lse2.stride(0), lse2.stride(1),
BLOCK_T=BLOCK_T, BLOCK_D=BLOCK_D)
return dsink
# --- forward as an opaque, fake-backed custom op -----------------------------------------
@torch.library.custom_op("olmo3_sink::fa3_sink_fwd", mutates_args=())
def _fa3_sink_fwd(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, sink: torch.Tensor,
cu_q: torch.Tensor, cu_k: torch.Tensor, max_q: int, max_k: int,
scale: float, causal: bool, wl: int, wr: int,
) -> tuple[torch.Tensor, torch.Tensor]:
# patched op: `sink` is the trailing arg; kernel applies it in the epilogue.
out, lse, _, _ = torch.ops.flash_attn_3.fwd(
q, k, v,
None, None, None, None, # k_new, v_new, qv, out
cu_q, cu_k, None, # cu_seqlens q/k/k_new
None, None, # seqused q/k
max_q, max_k,
None, None, None, # page_table, kv_batch_idx, leftpad_k
None, None, None, # rotary cos/sin, seqlens_rotary
None, None, None, # q/k/v_descale
scale, causal, wl, wr,
0, # attention_chunk
0.0, # softcap
False, # rotary_interleaved
None, # scheduler_metadata
1, # num_splits (DISABLE_SPLIT -> must be 1)
None, # pack_gqa
0, # sm_margin
sink, # proof-pilot sink
)
return out, lse
@_fa3_sink_fwd.register_fake
def _(q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr):
# out matches q [total_q, H, D]; varlen lse is [H, total_q] (see _dsink indexing).
T, H = q.shape[0], q.shape[1]
return torch.empty_like(q), q.new_empty((H, T), dtype=torch.float32)
# --- backward as an opaque, fake-backed custom op ----------------------------------------
@torch.library.custom_op("olmo3_sink::fa3_sink_bwd", mutates_args=())
def _fa3_sink_bwd(
do: torch.Tensor, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
out: torch.Tensor, lse: torch.Tensor, sink: torch.Tensor,
cu_q: torch.Tensor, cu_k: torch.Tensor, max_q: int, max_k: int,
scale: float, causal: bool, wl: int, wr: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
dq, dk, dv = torch.empty_like(q), torch.empty_like(k), torch.empty_like(v)
_flash_attn_backward(do, q, k, v, out, lse, cu_q, cu_k, None, None,
max_q, max_k, dq, dk, dv, scale, causal, wl, wr, 0.0, False, 0)
dsink = _dsink(out, do, sink, lse)
return dq, dk, dv, dsink
@_fa3_sink_bwd.register_fake
def _(do, q, k, v, out, lse, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr):
return (torch.empty_like(q), torch.empty_like(k), torch.empty_like(v),
torch.empty_like(sink))
def _setup_context(ctx, inputs, output):
q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr = inputs
out, lse = output
ctx.save_for_backward(q, k, v, out, lse, sink, cu_q, cu_k)
ctx.max_q, ctx.max_k = max_q, max_k
ctx.scale, ctx.causal, ctx.wl, ctx.wr = scale, causal, wl, wr
def _backward(ctx, grad_out, grad_lse):
q, k, v, out, lse, sink, cu_q, cu_k = ctx.saved_tensors
dq, dk, dv, dsink = torch.ops.olmo3_sink.fa3_sink_bwd(
grad_out.contiguous(), q, k, v, out, lse, sink, cu_q, cu_k,
ctx.max_q, ctx.max_k, ctx.scale, ctx.causal, ctx.wl, ctx.wr)
# grads for (q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr)
return dq, dk, dv, dsink, None, None, None, None, None, None, None, None
_fa3_sink_fwd.register_autograd(_backward, setup_context=_setup_context)
def fa3_varlen_attn_with_sink_kernel(
q, k, v, sink, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k,
softmax_scale=None, causal=True, window_size=(-1, -1),
):
if softmax_scale is None:
softmax_scale = q.shape[-1] ** -0.5
wl, wr = window_size
out, _lse = _fa3_sink_fwd(
q, k, v, sink.to(torch.float32), cu_seqlens_q, cu_seqlens_k,
int(max_seqlen_q), int(max_seqlen_k), float(softmax_scale), bool(causal), int(wl), int(wr))
return out