"""Reuse pinned FA3 unpadding metadata within one owned decode forward.""" from __future__ import annotations from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar import threading from typing import Any import torch from transformers.integrations.flash_attention import flash_attention_forward from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis _PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0" _VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"}) class _Entry: def __init__(self) -> None: self.key: tuple[Any, ...] | None = None self.mask: torch.Tensor | None = None self.indices: torch.Tensor | None = None self.cu_k: torch.Tensor | None = None self.cu_q: torch.Tensor | None = None self.maximum = 0 self.bytes = 0 def clear(self) -> None: self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None self.maximum, self.bytes = 0, 0 class DecodeMetadata: def __init__(self, max_bytes: int = 8 * 1024**2) -> None: if max_bytes <= 0: raise ValueError("Expected a positive metadata bound") self.max_bytes = max_bytes self._lock = threading.Lock() self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None) self._active: _Entry | None = None self._closed = False self.scopes = 0 self.materializations = 0 self.hits = 0 self.peak_bytes = 0 @contextmanager def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]: token = self._current.set(None) acquired = False entry = None try: inputs = kwargs.get("input_ids", args[0] if args else None) if inputs is None: inputs = kwargs.get("inputs_embeds") eligible = (not self._closed and not torch.is_grad_enabled() and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling() and isinstance(inputs, torch.Tensor) and inputs.is_cuda and inputs.ndim in (2, 3) and inputs.shape[1] == 1) if eligible and self._lock.acquire(blocking=False): acquired = True if not self._closed: entry = _Entry() self._active = entry self._current.set(entry) self.scopes += 1 yield finally: if entry is not None: entry.clear() self._active = None self._current.reset(token) if acquired: self._lock.release() def _metadata(self, mask: torch.Tensor) -> _Entry | None: entry = self._current.get() if self._closed or entry is None: return None batch = mask.shape[0] if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes: entry.clear() return None try: version = mask._version except RuntimeError: version = None key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride()), torch.cuda.current_stream(mask.device).cuda_stream) if entry.mask is mask and entry.key == key: self.hits += 1 return entry entry.clear() indices, cu_k, maximum = _get_unpad_data(mask) cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device) retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q)) if retained > self.max_bytes: return None entry.key, entry.mask = key, mask entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q entry.maximum, entry.bytes = int(maximum.item()), retained self.materializations += 1 self.peak_bytes = max(self.peak_bytes, retained) return entry def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]], module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.Tensor | None, *, dropout: float, scaling: float, **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]: eligible = (not self._closed and self._current.get() is not None and not torch.is_grad_enabled() and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling() and interface is flash_attention_forward and module.config._attn_implementation == _PINNED_FA3 and dropout == 0.0 and getattr(module.config, "sliding_window", None) is None and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys()) and mask is not None and mask.is_cuda and mask.ndim == 2 and mask.dtype == torch.bool and mask.numel() > 0 and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4 and item.device == mask.device and item.stride(-1) == 1 for item in (query, key, value))) if eligible: assert mask is not None positions = kwargs.get("position_ids") eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0] and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1] and mask.shape[1] > 1 and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4 and query.shape[3] == key.shape[3] == value.shape[3] == 256 and (positions is None or (isinstance(positions, torch.Tensor) and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1)))) entry = self._metadata(mask) if eligible and mask is not None else None if entry is None: active = self._current.get() if active is not None: active.clear() return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs) assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2) packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2) return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling, cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k, max_length_q=1, max_length_k=entry.maximum, **kwargs) def snapshot(self) -> dict[str, int | bool]: return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits, "peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0, "closed": self._closed} def close(self) -> None: self._closed = True with self._lock: if self._active is not None: self._active.clear() self._active = None