"""Share read-only recurrent packing metadata during one inference forward.""" from __future__ import annotations from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar from typing import Any import torch class _Forward: def __init__(self, hook_owner: torch.nn.Module | None = None) -> None: self.hook_owner = hook_owner self.unversioned_mask_allowed: bool | None = None self.hook_checks = 0 self.key: tuple[Any, ...] | None = None self.mask: torch.Tensor | None = None self.packed: tuple[torch.Tensor, torch.Tensor] | None = None self.slots_key: tuple[Any, ...] | None = None self.slots: torch.Tensor | None = None self.materializations = 0 self.hits = 0 self.slots_materializations = 0 self.slots_hits = 0 def clear(self) -> None: self.key, self.mask, self.packed = None, None, None self.slots_key, self.slots = None, None self.hook_owner, self.unversioned_mask_allowed = None, None _current: ContextVar[_Forward | None] = ContextVar("clairic_recurrent_metadata", default=None) _MAX_BYTES = 8 * 1024**2 @contextmanager def scope(hook_owner: torch.nn.Module | None = None) -> Iterator[None]: """Drop all added references on success, cancellation, or model failure.""" state = _Forward(hook_owner) token = _current.set(state) try: yield finally: state.clear() _current.reset(token) def _eligible(mixed: torch.Tensor) -> _Forward | None: state = _current.get() if (state is None or not mixed.is_cuda or mixed.dtype != torch.bfloat16 or mixed.ndim != 3 or torch.is_grad_enabled() or not torch.is_inference_mode_enabled() or torch.compiler.is_compiling()): return None return state def _no_forward_hooks(owner: torch.nn.Module | None) -> bool: """Admit only a known hook-free tree and hook-free global registries.""" if not isinstance(owner, torch.nn.Module): return False registry = torch.nn.modules.module for name in ("_global_forward_pre_hooks", "_global_forward_hooks"): hooks = getattr(registry, name, None) if hooks is None or hooks: return False for module in owner.modules(): for name in ("_forward_pre_hooks", "_forward_hooks"): hooks = getattr(module, name, None) if hooks is None or hooks: return False return True def packed_indices(mixed: torch.Tensor, mask: torch.Tensor | None, original: Callable[[torch.Tensor, torch.Tensor | None], tuple[torch.Tensor, torch.Tensor]] ) -> tuple[torch.Tensor, torch.Tensor]: """Reuse the original helper's exact results for an unchanged current mask.""" state = _eligible(mixed) batch, length = mixed.shape[:2] required = batch * length * 8 + (2 * batch + 1) * 4 if state is None or required > _MAX_BYTES: return original(mixed, mask) stream = torch.cuda.current_stream(mixed.device).cuda_stream mask_key: tuple[Any, ...] = () if mask is not None: try: version = mask._version except RuntimeError: version = None if version is None: if state.unversioned_mask_allowed is None: state.hook_checks += 1 state.unversioned_mask_allowed = _no_forward_hooks(state.hook_owner) if not state.unversioned_mask_allowed: state.key, state.mask, state.packed = None, None, None return original(mixed, mask) mask_key = (id(mask), version, mask.data_ptr(), mask.storage_offset(), mask.device, mask.dtype, tuple(mask.shape), tuple(mask.stride())) key = (batch, length, mixed.device, stream, mask_key) if state.key == key and state.mask is mask and state.packed is not None: state.hits += 1 return state.packed state.key, state.mask, state.packed = None, None, None if state.slots_key != (batch, mixed.device, stream): state.slots_key, state.slots = None, None packed = original(mixed, mask) retained = sum(value.numel() * value.element_size() for value in packed) if retained + batch * 4 <= _MAX_BYTES: state.key, state.mask, state.packed = key, mask, packed state.materializations += 1 return packed def sequence_slots(mixed: torch.Tensor) -> torch.Tensor: """Reuse the same identity slot vector in each recurrent layer.""" state = _eligible(mixed) batch = mixed.shape[0] retained = (sum(value.numel() * value.element_size() for value in state.packed) if state is not None and state.packed is not None else 0) if state is None or retained + batch * 4 > _MAX_BYTES: return torch.arange(batch, dtype=torch.int32, device=mixed.device) key = (batch, mixed.device, torch.cuda.current_stream(mixed.device).cuda_stream) if state.slots_key != key or state.slots is None: state.slots = torch.arange(batch, dtype=torch.int32, device=mixed.device) state.slots_key = key state.slots_materializations += 1 else: state.slots_hits += 1 return state.slots