File size: 7,605 Bytes
524fd9d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
"""Reuse pinned FA3 unpadding metadata within one owned decode forward."""
from __future__ import annotations

from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
import threading
from typing import Any

import torch
from transformers.integrations.flash_attention import flash_attention_forward
from transformers.modeling_flash_attention_utils import _get_unpad_data, _index_first_axis

_PINNED_FA3 = "kernels-community/flash-attn3@e29f138fc363b396e5d2706c8a5f6fa7d36f41e0"
_VARLEN_KEYS = frozenset({"cu_seq_lens_q", "cu_seq_lens_k", "max_length_q", "max_length_k"})


class _Entry:
    def __init__(self) -> None:
        self.key: tuple[Any, ...] | None = None
        self.mask: torch.Tensor | None = None
        self.indices: torch.Tensor | None = None
        self.cu_k: torch.Tensor | None = None
        self.cu_q: torch.Tensor | None = None
        self.maximum = 0
        self.bytes = 0

    def clear(self) -> None:
        self.key, self.mask, self.indices, self.cu_k, self.cu_q = None, None, None, None, None
        self.maximum, self.bytes = 0, 0


class DecodeMetadata:
    def __init__(self, max_bytes: int = 8 * 1024**2) -> None:
        if max_bytes <= 0:
            raise ValueError("Expected a positive metadata bound")
        self.max_bytes = max_bytes
        self._lock = threading.Lock()
        self._current: ContextVar[_Entry | None] = ContextVar("clairic_decode_metadata", default=None)
        self._active: _Entry | None = None
        self._closed = False
        self.scopes = 0
        self.materializations = 0
        self.hits = 0
        self.peak_bytes = 0

    @contextmanager
    def scope(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Iterator[None]:
        token = self._current.set(None)
        acquired = False
        entry = None
        try:
            inputs = kwargs.get("input_ids", args[0] if args else None)
            if inputs is None:
                inputs = kwargs.get("inputs_embeds")
            eligible = (not self._closed and not torch.is_grad_enabled()
                        and torch.is_inference_mode_enabled() and not torch.compiler.is_compiling()
                        and isinstance(inputs, torch.Tensor) and inputs.is_cuda
                        and inputs.ndim in (2, 3) and inputs.shape[1] == 1)
            if eligible and self._lock.acquire(blocking=False):
                acquired = True
                if not self._closed:
                    entry = _Entry()
                    self._active = entry
                    self._current.set(entry)
                    self.scopes += 1
            yield
        finally:
            if entry is not None:
                entry.clear()
                self._active = None
            self._current.reset(token)
            if acquired:
                self._lock.release()

    def _metadata(self, mask: torch.Tensor) -> _Entry | None:
        entry = self._current.get()
        if self._closed or entry is None:
            return None
        batch = mask.shape[0]
        if mask.numel() * 8 + 2 * (batch + 1) * 4 > self.max_bytes:
            entry.clear()
            return None
        try:
            version = mask._version
        except RuntimeError:
            version = None
        key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device,
               mask.dtype, tuple(mask.shape), tuple(mask.stride()),
               torch.cuda.current_stream(mask.device).cuda_stream)
        if entry.mask is mask and entry.key == key:
            self.hits += 1
            return entry
        entry.clear()
        indices, cu_k, maximum = _get_unpad_data(mask)
        cu_q = torch.arange(batch + 1, dtype=torch.int32, device=mask.device)
        retained = sum(value.numel() * value.element_size() for value in (indices, cu_k, cu_q))
        if retained > self.max_bytes:
            return None
        entry.key, entry.mask = key, mask
        entry.indices, entry.cu_k, entry.cu_q = indices, cu_k, cu_q
        entry.maximum, entry.bytes = int(maximum.item()), retained
        self.materializations += 1
        self.peak_bytes = max(self.peak_bytes, retained)
        return entry

    def attend(self, interface: Callable[..., tuple[torch.Tensor, torch.Tensor | None]],
               module: Any, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
               mask: torch.Tensor | None, *, dropout: float, scaling: float,
               **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
        eligible = (not self._closed and self._current.get() is not None
                    and not torch.is_grad_enabled() and torch.is_inference_mode_enabled()
                    and not torch.compiler.is_compiling() and interface is flash_attention_forward
                    and module.config._attn_implementation == _PINNED_FA3
                    and dropout == 0.0
                    and getattr(module.config, "sliding_window", None) is None
                    and kwargs.get("sliding_window") is None and not (_VARLEN_KEYS & kwargs.keys())
                    and mask is not None and mask.is_cuda and mask.ndim == 2
                    and mask.dtype == torch.bool and mask.numel() > 0
                    and all(item.is_cuda and item.dtype == torch.bfloat16 and item.ndim == 4
                            and item.device == mask.device and item.stride(-1) == 1
                            for item in (query, key, value)))
        if eligible:
            assert mask is not None
            positions = kwargs.get("position_ids")
            eligible = (query.shape[0] == key.shape[0] == value.shape[0] == mask.shape[0]
                        and query.shape[2] == 1 and key.shape[2] == value.shape[2] == mask.shape[1]
                        and mask.shape[1] > 1
                        and query.shape[1] in (16, 24) and key.shape[1] == value.shape[1] == 4
                        and query.shape[3] == key.shape[3] == value.shape[3] == 256
                        and (positions is None or (isinstance(positions, torch.Tensor)
                             and positions.ndim == 2 and tuple(positions.shape) == (query.shape[0], 1))))
        entry = self._metadata(mask) if eligible and mask is not None else None
        if entry is None:
            active = self._current.get()
            if active is not None:
                active.clear()
            return interface(module, query, key, value, mask, dropout=dropout, scaling=scaling, **kwargs)
        assert entry.indices is not None and entry.cu_q is not None and entry.cu_k is not None
        packed_key = _index_first_axis(key.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
        packed_value = _index_first_axis(value.transpose(1, 2), entry.indices).unsqueeze(0).transpose(1, 2)
        return interface(module, query, packed_key, packed_value, None, dropout=dropout, scaling=scaling,
                         cu_seq_lens_q=entry.cu_q, cu_seq_lens_k=entry.cu_k,
                         max_length_q=1, max_length_k=entry.maximum, **kwargs)

    def snapshot(self) -> dict[str, int | bool]:
        return {"scopes": self.scopes, "materializations": self.materializations, "hits": self.hits,
                "peak_bytes": self.peak_bytes, "retained_bytes": self._active.bytes if self._active else 0,
                "closed": self._closed}

    def close(self) -> None:
        self._closed = True
        with self._lock:
            if self._active is not None:
                self._active.clear()
                self._active = None