"""Reviewed reference copy; CPU/CUDA input guard only changed for integration.""" from dataclasses import dataclass import math import torch from torch import nn from torch.nn import functional as F @dataclass(frozen=True) class TTTConfig: dim: int = 4 memory_dim: int = 3 hidden_dim: int = 6 bias: bool = True reduction: str = 'mean' # per-trajectory token/feature mean, or sum base_lr: float = 0.1 lr_parameterization: str = 'softplus' # or exp; multiplier starts at one output_projection: bool = True def __post_init__(self): if min(self.dim, self.memory_dim, self.hidden_dim) <= 0: raise ValueError('Dimensions must be positive') if self.reduction not in ('mean', 'sum'): raise ValueError('reduction must be mean or sum') if self.lr_parameterization not in ('softplus', 'exp') or self.base_lr <= 0: raise ValueError('Invalid learning rate configuration') if not self.output_projection and self.dim != self.memory_dim: raise ValueError('Without output projection, dim must equal memory_dim') @dataclass(frozen=True) class FastState: weights: tuple[torch.Tensor, ...] updates: torch.Tensor # [B], valid observation updates, not token positions def detach(self): # Detach the complete W, NOT a delta followed by adding W0. return FastState(tuple(w.detach() for w in self.weights), self.updates.detach()) class TTTLayer(nn.Module): def __init__(self, config: TTTConfig): super().__init__() self.config = config d, r, h = config.dim, config.memory_dim, config.hidden_dim self.q = nn.Linear(d, r, bias=False) self.k = nn.Linear(d, r, bias=False) self.v = nn.Linear(d, r, bias=False) self.out = nn.Linear(r, d, bias=False) if config.output_projection else nn.Identity() w1, w2 = torch.empty(r, h), torch.empty(h, r) nn.init.xavier_uniform_(w1) nn.init.xavier_uniform_(w2) values = [w1, w2] if config.bias: values += [torch.zeros(h), torch.zeros(r)] self.w0 = nn.ParameterList([nn.Parameter(w) for w in values]) init = math.log(math.expm1(1.0)) if config.lr_parameterization == 'softplus' else 0.0 self.raw_lr = nn.Parameter(torch.tensor(init)) self.alpha = nn.Parameter(torch.full((d,), 0.001)) @property def inner_lr(self): multiplier = F.softplus(self.raw_lr) if self.config.lr_parameterization == 'softplus' else self.raw_lr.exp() return self.config.base_lr * multiplier def forward(self, x, state, **kwargs): return self.step(x, state, **kwargs) def initial_state(self, batch_size: int): if batch_size <= 0: raise ValueError('batch_size must be positive') # Clone gives independent storage while preserving the path to learned W0. return FastState(tuple(w.unsqueeze(0).expand(batch_size, *w.shape).clone() for w in self.w0), torch.zeros(batch_size, dtype=torch.long, device=self.alpha.device)) def reset(self, state: FastState, reset_mask: torch.Tensor): b = state.updates.shape[0] if reset_mask.shape != (b,) or reset_mask.dtype != torch.bool: raise ValueError('reset_mask must be bool [B]') # A no-op reset must not attach W0 with a zero gradient: optimizers can # still apply momentum/weight decay when grad is zero rather than None. if not bool(reset_mask.any()): return state fresh = self.initial_state(b) return FastState(tuple(torch.where(reset_mask.reshape(b, *([1] * (w.ndim-1))), w0, w) for w, w0 in zip(state.weights, fresh.weights)), torch.where(reset_mask, fresh.updates, state.updates)) @staticmethod def fast(z, weights): w1, w2 = weights[:2] hidden = z @ w1 if len(weights) == 4: hidden = hidden + weights[2][:, None, :] y = F.gelu(hidden, approximate='none') @ w2 return y + weights[3][:, None, :] if len(weights) == 4 else y def step(self, x, state: FastState, *, token_mask=None, timestep_mask=None, differentiable=True, inner_trace=None): """Update all tokens once from incoming W; then read Q from updated W. differentiable=False explicitly detaches incoming/outgoing state and computes only inner gradients, even inside torch.no_grad(). An enclosing torch.inference_mode() is rejected because inference tensors cannot take part in the required autograd operation. """ if torch.is_inference_mode_enabled(): raise RuntimeError('TTT requires inner autograd; use no_grad, not inference_mode') if x.ndim != 3 or x.shape[-1] != self.config.dim or x.device.type not in ('cpu', 'cuda'): raise ValueError('Expected CPU/CUDA [B,S,D]') b, s, _ = x.shape if b <= 0 or s <= 0: raise ValueError('Empty batch/token axis is unsupported; mask padding instead') if state.updates.shape != (b,) or len(state.weights) != len(self.w0): raise ValueError('Incompatible state') for w, w0 in zip(state.weights, self.w0): if w.shape != (b, *w0.shape) or w.dtype != x.dtype or w.device != x.device: raise ValueError('State shape/dtype/device mismatch') if x.dtype != self.alpha.dtype: raise ValueError('Input/model dtype mismatch') valid = torch.ones((b, s), dtype=torch.bool, device=x.device) for mask, shape in ((token_mask, (b, s)), (timestep_mask, (b,))): if mask is not None and (mask.shape != shape or mask.dtype != torch.bool): raise ValueError('Masks must have bool token [B,S] / timestep [B] shapes') if token_mask is not None: valid = valid & token_mask if timestep_mask is not None: valid = valid & timestep_mask[:, None] with torch.enable_grad(): # Include sanitizing in the explicit grad region so an enclosing # no_grad does not silently cut the differentiable input branch. clean = torch.where(valid[..., None], x, torch.zeros_like(x)) weights = tuple(w if w.requires_grad else w.detach().requires_grad_(True) for w in (state.weights if differentiable else state.detach().weights)) # Local weight arguments isolate partial_W; clone preserves ALL outer paths. weights = tuple(w.clone() for w in weights) if differentiable: q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr else: with torch.no_grad(): q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr err = (self.fast(k, weights) - v).square() per_trajectory = torch.where(valid[..., None], err, 0).sum((1, 2)) if self.config.reduction == 'mean': per_trajectory = per_trajectory / (valid.sum(1) * self.config.memory_dim).clamp_min(1) # Sum across independent trajectories; never divide by batch size. grads = torch.autograd.grad(per_trajectory.sum(), weights, create_graph=differentiable) if inner_trace is not None: inner_trace.update({'grads':[g.detach().cpu().clone() for g in grads], 'q':q.detach().cpu().clone(),'k':k.detach().cpu().clone(),'v':v.detach().cpu().clone()}) updated = tuple(w - lr * g for w, g in zip(weights, grads)) next_state = FastState(updated, state.updates + valid.any(1).long()) if differentiable: memory = self.out(self.fast(q, updated)) y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0) else: with torch.no_grad(): memory = self.out(self.fast(q, updated)) y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0) next_state = next_state.detach() return y, next_state