clairic / clairic_runtime /recurrent_metadata.py
areneau's picture
Update Clairic Transformers runtime
524fd9d verified
Raw History Blame Contribute Delete
5.32 kB
"""Share read-only recurrent packing metadata during one inference forward."""
from __future__ import annotations
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any
import torch
class _Forward:
def __init__(self, hook_owner: torch.nn.Module | None = None) -> None:
self.hook_owner = hook_owner
self.unversioned_mask_allowed: bool | None = None
self.hook_checks = 0
self.key: tuple[Any, ...] | None = None
self.mask: torch.Tensor | None = None
self.packed: tuple[torch.Tensor, torch.Tensor] | None = None
self.slots_key: tuple[Any, ...] | None = None
self.slots: torch.Tensor | None = None
self.materializations = 0
self.hits = 0
self.slots_materializations = 0
self.slots_hits = 0
def clear(self) -> None:
self.key, self.mask, self.packed = None, None, None
self.slots_key, self.slots = None, None
self.hook_owner, self.unversioned_mask_allowed = None, None
_current: ContextVar[_Forward | None] = ContextVar("clairic_recurrent_metadata", default=None)
_MAX_BYTES = 8 * 1024**2
@contextmanager
def scope(hook_owner: torch.nn.Module | None = None) -> Iterator[None]:
"""Drop all added references on success, cancellation, or model failure."""
state = _Forward(hook_owner)
token = _current.set(state)
try:
yield
finally:
state.clear()
_current.reset(token)
def _eligible(mixed: torch.Tensor) -> _Forward | None:
state = _current.get()
if (state is None or not mixed.is_cuda or mixed.dtype != torch.bfloat16
or mixed.ndim != 3 or torch.is_grad_enabled()
or not torch.is_inference_mode_enabled() or torch.compiler.is_compiling()):
return None
return state
def _no_forward_hooks(owner: torch.nn.Module | None) -> bool:
"""Admit only a known hook-free tree and hook-free global registries."""
if not isinstance(owner, torch.nn.Module):
return False
registry = torch.nn.modules.module
for name in ("_global_forward_pre_hooks", "_global_forward_hooks"):
hooks = getattr(registry, name, None)
if hooks is None or hooks:
return False
for module in owner.modules():
for name in ("_forward_pre_hooks", "_forward_hooks"):
hooks = getattr(module, name, None)
if hooks is None or hooks:
return False
return True
def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None,
original: Callable[[torch.Tensor, torch.Tensor | None],
tuple[torch.Tensor, torch.Tensor]]
) -> tuple[torch.Tensor, torch.Tensor]:
"""Reuse the original helper's exact results for an unchanged current mask."""
state = _eligible(mixed)
batch, length = mixed.shape[:2]
required = batch * length * 8 + (2 * batch + 1) * 4
if state is None or required > _MAX_BYTES:
return original(mixed, mask)
stream = torch.cuda.current_stream(mixed.device).cuda_stream
mask_key: tuple[Any, ...] = ()
if mask is not None:
try:
version = mask._version
except RuntimeError:
version = None
if version is None:
if state.unversioned_mask_allowed is None:
state.hook_checks += 1
state.unversioned_mask_allowed = _no_forward_hooks(state.hook_owner)
if not state.unversioned_mask_allowed:
state.key, state.mask, state.packed = None, None, None
return original(mixed, mask)
mask_key = (id(mask), version, mask.data_ptr(), mask.storage_offset(),
mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride()))
key = (batch, length, mixed.device, stream, mask_key)
if state.key == key and state.mask is mask and state.packed is not None:
state.hits += 1
return state.packed
state.key, state.mask, state.packed = None, None, None
if state.slots_key != (batch, mixed.device, stream):
state.slots_key, state.slots = None, None
packed = original(mixed, mask)
retained = sum(value.numel() * value.element_size() for value in packed)
if retained + batch * 4 <= _MAX_BYTES:
state.key, state.mask, state.packed = key, mask, packed
state.materializations += 1
return packed
def sequence_slots(mixed: torch.Tensor) -> torch.Tensor:
"""Reuse the same identity slot vector in each recurrent layer."""
state = _eligible(mixed)
batch = mixed.shape[0]
retained = (sum(value.numel() * value.element_size() for value in state.packed)
if state is not None and state.packed is not None else 0)
if state is None or retained + batch * 4 > _MAX_BYTES:
return torch.arange(batch, dtype=torch.int32, device=mixed.device)
key = (batch, mixed.device, torch.cuda.current_stream(mixed.device).cuda_stream)
if state.slots_key != key or state.slots is None:
state.slots = torch.arange(batch, dtype=torch.int32, device=mixed.device)
state.slots_key = key
state.slots_materializations += 1
else:
state.slots_hits += 1
return state.slots