Download step02-olmo32b-deepseektok-sink/fa3_sink_kernel.py from fieldsmodelorg/proof-pilot-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 6.77 kB
-
https://huggingface.co/fieldsmodelorg/proof-pilot-checkpoints/resolve/main/step02-olmo32b-deepseektok-sink/fa3_sink_kernel.py
- Command line
-
hf download hf://fieldsmodelorg/proof-pilot-checkpoints/step02-olmo32b-deepseektok-sink/fa3_sink_kernel.py
-
curl -L -o fa3_sink_kernel.py https://huggingface.co/fieldsmodelorg/proof-pilot-checkpoints/resolve/main/step02-olmo32b-deepseektok-sink/fa3_sink_kernel.py
6.77 kB
| # Copyright 2026 proof-pilot. Apache-2.0. | |
| """In-kernel attention sink on patched FA3 (targets >95% of no-sink FA3 for ALL packing). | |
| The FA3 forward kernel epilogue is patched (see flash-attention/hopper/softmax.h finalize) | |
| to add exp(sink-M) to the softmax denominator, so it directly outputs sink-corrected `o` | |
| and sink-inclusive `lse` — NO forward post-pass. Backward feeds those to FA3's native | |
| backward for exact dq/dk/dv (proven equivalent), plus a cheap dsink reduction. | |
| torch.compile: forward and backward are each a `torch.library.custom_op` with a registered | |
| fake (meta) impl, so Dynamo treats them as single opaque nodes with known output shapes | |
| instead of graph-breaking on the fake-less `flash_attn_3.fwd`. The FA3/Triton internals | |
| live in the eager op bodies and are never traced. This lets a compiled decoder layer fuse | |
| everything around attention into one graph (critical for compile + gradient checkpointing, | |
| which re-runs the forward in backward and would otherwise pay the graph-break cost twice). | |
| """ | |
| from __future__ import annotations | |
| import torch | |
| import triton | |
| import triton.language as tl | |
| from flash_attn_interface import _flash_attn_backward | |
| def _dsink_kernel(o_ptr, do_ptr, sink_ptr, lse2_ptr, dsink_ptr, T, D, | |
| so_t, so_h, sl_h, sl_t, BLOCK_T: tl.constexpr, BLOCK_D: tl.constexpr): | |
| h = tl.program_id(0) | |
| t = tl.program_id(1) * BLOCK_T + tl.arange(0, BLOCK_T) | |
| d = tl.arange(0, BLOCK_D) | |
| tm = t < T | |
| dm = d < D | |
| base = t[:, None] * so_t + h * so_h + d[None, :] | |
| m = tm[:, None] & dm[None, :] | |
| o = tl.load(o_ptr + base, mask=m, other=0.0).to(tl.float32) | |
| g = tl.load(do_ptr + base, mask=m, other=0.0).to(tl.float32) | |
| delta = tl.sum(o * g, axis=1) # [BLOCK_T] | |
| lse2 = tl.load(lse2_ptr + h * sl_h + t * sl_t, mask=tm, other=0.0) | |
| sink = tl.load(sink_ptr + h) | |
| contrib = tl.where(tm, -tl.exp(sink - lse2) * delta, 0.0) | |
| tl.atomic_add(dsink_ptr + h, tl.sum(contrib)) | |
| def _dsink(o_sink, do, sink, lse2): | |
| """dsink_h = -Σ_t exp(sink_h - lse2_{h,t})·(o_sink·do)_{t,h}. o_sink,do [T,H,D]; lse2 [H,T].""" | |
| T, H, D = o_sink.shape | |
| dsink = torch.zeros(H, device=o_sink.device, dtype=torch.float32) | |
| BLOCK_T = 64 | |
| BLOCK_D = triton.next_power_of_2(D) | |
| grid = (H, triton.cdiv(T, BLOCK_T)) | |
| _dsink_kernel[grid](o_sink, do, sink, lse2, dsink, T, D, | |
| o_sink.stride(0), o_sink.stride(1), lse2.stride(0), lse2.stride(1), | |
| BLOCK_T=BLOCK_T, BLOCK_D=BLOCK_D) | |
| return dsink | |
| # --- forward as an opaque, fake-backed custom op ----------------------------------------- | |
| def _fa3_sink_fwd( | |
| q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, sink: torch.Tensor, | |
| cu_q: torch.Tensor, cu_k: torch.Tensor, max_q: int, max_k: int, | |
| scale: float, causal: bool, wl: int, wr: int, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| # patched op: `sink` is the trailing arg; kernel applies it in the epilogue. | |
| out, lse, _, _ = torch.ops.flash_attn_3.fwd( | |
| q, k, v, | |
| None, None, None, None, # k_new, v_new, qv, out | |
| cu_q, cu_k, None, # cu_seqlens q/k/k_new | |
| None, None, # seqused q/k | |
| max_q, max_k, | |
| None, None, None, # page_table, kv_batch_idx, leftpad_k | |
| None, None, None, # rotary cos/sin, seqlens_rotary | |
| None, None, None, # q/k/v_descale | |
| scale, causal, wl, wr, | |
| 0, # attention_chunk | |
| 0.0, # softcap | |
| False, # rotary_interleaved | |
| None, # scheduler_metadata | |
| 1, # num_splits (DISABLE_SPLIT -> must be 1) | |
| None, # pack_gqa | |
| 0, # sm_margin | |
| sink, # proof-pilot sink | |
| ) | |
| return out, lse | |
| def _(q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr): | |
| # out matches q [total_q, H, D]; varlen lse is [H, total_q] (see _dsink indexing). | |
| T, H = q.shape[0], q.shape[1] | |
| return torch.empty_like(q), q.new_empty((H, T), dtype=torch.float32) | |
| # --- backward as an opaque, fake-backed custom op ---------------------------------------- | |
| def _fa3_sink_bwd( | |
| do: torch.Tensor, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, | |
| out: torch.Tensor, lse: torch.Tensor, sink: torch.Tensor, | |
| cu_q: torch.Tensor, cu_k: torch.Tensor, max_q: int, max_k: int, | |
| scale: float, causal: bool, wl: int, wr: int, | |
| ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: | |
| dq, dk, dv = torch.empty_like(q), torch.empty_like(k), torch.empty_like(v) | |
| _flash_attn_backward(do, q, k, v, out, lse, cu_q, cu_k, None, None, | |
| max_q, max_k, dq, dk, dv, scale, causal, wl, wr, 0.0, False, 0) | |
| dsink = _dsink(out, do, sink, lse) | |
| return dq, dk, dv, dsink | |
| def _(do, q, k, v, out, lse, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr): | |
| return (torch.empty_like(q), torch.empty_like(k), torch.empty_like(v), | |
| torch.empty_like(sink)) | |
| def _setup_context(ctx, inputs, output): | |
| q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr = inputs | |
| out, lse = output | |
| ctx.save_for_backward(q, k, v, out, lse, sink, cu_q, cu_k) | |
| ctx.max_q, ctx.max_k = max_q, max_k | |
| ctx.scale, ctx.causal, ctx.wl, ctx.wr = scale, causal, wl, wr | |
| def _backward(ctx, grad_out, grad_lse): | |
| q, k, v, out, lse, sink, cu_q, cu_k = ctx.saved_tensors | |
| dq, dk, dv, dsink = torch.ops.olmo3_sink.fa3_sink_bwd( | |
| grad_out.contiguous(), q, k, v, out, lse, sink, cu_q, cu_k, | |
| ctx.max_q, ctx.max_k, ctx.scale, ctx.causal, ctx.wl, ctx.wr) | |
| # grads for (q, k, v, sink, cu_q, cu_k, max_q, max_k, scale, causal, wl, wr) | |
| return dq, dk, dv, dsink, None, None, None, None, None, None, None, None | |
| _fa3_sink_fwd.register_autograd(_backward, setup_context=_setup_context) | |
| def fa3_varlen_attn_with_sink_kernel( | |
| q, k, v, sink, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, | |
| softmax_scale=None, causal=True, window_size=(-1, -1), | |
| ): | |
| if softmax_scale is None: | |
| softmax_scale = q.shape[-1] ** -0.5 | |
| wl, wr = window_size | |
| out, _lse = _fa3_sink_fwd( | |
| q, k, v, sink.to(torch.float32), cu_seqlens_q, cu_seqlens_k, | |
| int(max_seqlen_q), int(max_seqlen_k), float(softmax_scale), bool(causal), int(wl), int(wr)) | |
| return out | |