File size: 5,319 Bytes
24191a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
"""Share read-only recurrent packing metadata during one inference forward."""
from __future__ import annotations

from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any

import torch


class _Forward:
    def __init__(self, hook_owner: torch.nn.Module | None = None) -> None:
        self.hook_owner = hook_owner
        self.unversioned_mask_allowed: bool | None = None
        self.hook_checks = 0
        self.key: tuple[Any, ...] | None = None
        self.mask: torch.Tensor | None = None
        self.packed: tuple[torch.Tensor, torch.Tensor] | None = None
        self.slots_key: tuple[Any, ...] | None = None
        self.slots: torch.Tensor | None = None
        self.materializations = 0
        self.hits = 0
        self.slots_materializations = 0
        self.slots_hits = 0

    def clear(self) -> None:
        self.key, self.mask, self.packed = None, None, None
        self.slots_key, self.slots = None, None
        self.hook_owner, self.unversioned_mask_allowed = None, None


_current: ContextVar[_Forward | None] = ContextVar("clairic_recurrent_metadata", default=None)
_MAX_BYTES = 8 * 1024**2


@contextmanager
def scope(hook_owner: torch.nn.Module | None = None) -> Iterator[None]:
    """Drop all added references on success, cancellation, or model failure."""
    state = _Forward(hook_owner)
    token = _current.set(state)
    try:
        yield
    finally:
        state.clear()
        _current.reset(token)


def _eligible(mixed: torch.Tensor) -> _Forward | None:
    state = _current.get()
    if (state is None or not mixed.is_cuda or mixed.dtype != torch.bfloat16
            or mixed.ndim != 3 or torch.is_grad_enabled()
            or not torch.is_inference_mode_enabled() or torch.compiler.is_compiling()):
        return None
    return state


def _no_forward_hooks(owner: torch.nn.Module | None) -> bool:
    """Admit only a known hook-free tree and hook-free global registries."""
    if not isinstance(owner, torch.nn.Module):
        return False
    registry = torch.nn.modules.module
    for name in ("_global_forward_pre_hooks", "_global_forward_hooks"):
        hooks = getattr(registry, name, None)
        if hooks is None or hooks:
            return False
    for module in owner.modules():
        for name in ("_forward_pre_hooks", "_forward_hooks"):
            hooks = getattr(module, name, None)
            if hooks is None or hooks:
                return False
    return True


def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None,
                   original: Callable[[torch.Tensor, torch.Tensor | None],
                                      tuple[torch.Tensor, torch.Tensor]]
                   ) -> tuple[torch.Tensor, torch.Tensor]:
    """Reuse the original helper's exact results for an unchanged current mask."""
    state = _eligible(mixed)
    batch, length = mixed.shape[:2]
    required = batch * length * 8 + (2 * batch + 1) * 4
    if state is None or required > _MAX_BYTES:
        return original(mixed, mask)
    stream = torch.cuda.current_stream(mixed.device).cuda_stream
    mask_key: tuple[Any, ...] = ()
    if mask is not None:
        try:
            version = mask._version
        except RuntimeError:
            version = None
        if version is None:
            if state.unversioned_mask_allowed is None:
                state.hook_checks += 1
                state.unversioned_mask_allowed = _no_forward_hooks(state.hook_owner)
            if not state.unversioned_mask_allowed:
                state.key, state.mask, state.packed = None, None, None
                return original(mixed, mask)
        mask_key = (id(mask), version, mask.data_ptr(), mask.storage_offset(),
                    mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride()))
    key = (batch, length, mixed.device, stream, mask_key)
    if state.key == key and state.mask is mask and state.packed is not None:
        state.hits += 1
        return state.packed
    state.key, state.mask, state.packed = None, None, None
    if state.slots_key != (batch, mixed.device, stream):
        state.slots_key, state.slots = None, None
    packed = original(mixed, mask)
    retained = sum(value.numel() * value.element_size() for value in packed)
    if retained + batch * 4 <= _MAX_BYTES:
        state.key, state.mask, state.packed = key, mask, packed
        state.materializations += 1
    return packed


def sequence_slots(mixed: torch.Tensor) -> torch.Tensor:
    """Reuse the same identity slot vector in each recurrent layer."""
    state = _eligible(mixed)
    batch = mixed.shape[0]
    retained = (sum(value.numel() * value.element_size() for value in state.packed)
                if state is not None and state.packed is not None else 0)
    if state is None or retained + batch * 4 > _MAX_BYTES:
        return torch.arange(batch, dtype=torch.int32, device=mixed.device)
    key = (batch, mixed.device, torch.cuda.current_stream(mixed.device).cuda_stream)
    if state.slots_key != key or state.slots is None:
        state.slots = torch.arange(batch, dtype=torch.int32, device=mixed.device)
        state.slots_key = key
        state.slots_materializations += 1
    else:
        state.slots_hits += 1
    return state.slots