jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
8.3 kB
"""Reviewed reference copy; CPU/CUDA input guard only changed for integration."""
from dataclasses import dataclass
import math
import torch
from torch import nn
from torch.nn import functional as F
@dataclass(frozen=True)
class TTTConfig:
dim: int = 4
memory_dim: int = 3
hidden_dim: int = 6
bias: bool = True
reduction: str = 'mean' # per-trajectory token/feature mean, or sum
base_lr: float = 0.1
lr_parameterization: str = 'softplus' # or exp; multiplier starts at one
output_projection: bool = True
def __post_init__(self):
if min(self.dim, self.memory_dim, self.hidden_dim) <= 0:
raise ValueError('Dimensions must be positive')
if self.reduction not in ('mean', 'sum'):
raise ValueError('reduction must be mean or sum')
if self.lr_parameterization not in ('softplus', 'exp') or self.base_lr <= 0:
raise ValueError('Invalid learning rate configuration')
if not self.output_projection and self.dim != self.memory_dim:
raise ValueError('Without output projection, dim must equal memory_dim')
@dataclass(frozen=True)
class FastState:
weights: tuple[torch.Tensor, ...]
updates: torch.Tensor # [B], valid observation updates, not token positions
def detach(self):
# Detach the complete W, NOT a delta followed by adding W0.
return FastState(tuple(w.detach() for w in self.weights), self.updates.detach())
class TTTLayer(nn.Module):
def __init__(self, config: TTTConfig):
super().__init__()
self.config = config
d, r, h = config.dim, config.memory_dim, config.hidden_dim
self.q = nn.Linear(d, r, bias=False)
self.k = nn.Linear(d, r, bias=False)
self.v = nn.Linear(d, r, bias=False)
self.out = nn.Linear(r, d, bias=False) if config.output_projection else nn.Identity()
w1, w2 = torch.empty(r, h), torch.empty(h, r)
nn.init.xavier_uniform_(w1)
nn.init.xavier_uniform_(w2)
values = [w1, w2]
if config.bias:
values += [torch.zeros(h), torch.zeros(r)]
self.w0 = nn.ParameterList([nn.Parameter(w) for w in values])
init = math.log(math.expm1(1.0)) if config.lr_parameterization == 'softplus' else 0.0
self.raw_lr = nn.Parameter(torch.tensor(init))
self.alpha = nn.Parameter(torch.full((d,), 0.001))
@property
def inner_lr(self):
multiplier = F.softplus(self.raw_lr) if self.config.lr_parameterization == 'softplus' else self.raw_lr.exp()
return self.config.base_lr * multiplier
def forward(self, x, state, **kwargs):
return self.step(x, state, **kwargs)
def initial_state(self, batch_size: int):
if batch_size <= 0:
raise ValueError('batch_size must be positive')
# Clone gives independent storage while preserving the path to learned W0.
return FastState(tuple(w.unsqueeze(0).expand(batch_size, *w.shape).clone()
for w in self.w0),
torch.zeros(batch_size, dtype=torch.long, device=self.alpha.device))
def reset(self, state: FastState, reset_mask: torch.Tensor):
b = state.updates.shape[0]
if reset_mask.shape != (b,) or reset_mask.dtype != torch.bool:
raise ValueError('reset_mask must be bool [B]')
# A no-op reset must not attach W0 with a zero gradient: optimizers can
# still apply momentum/weight decay when grad is zero rather than None.
if not bool(reset_mask.any()):
return state
fresh = self.initial_state(b)
return FastState(tuple(torch.where(reset_mask.reshape(b, *([1] * (w.ndim-1))), w0, w)
for w, w0 in zip(state.weights, fresh.weights)),
torch.where(reset_mask, fresh.updates, state.updates))
@staticmethod
def fast(z, weights):
w1, w2 = weights[:2]
hidden = z @ w1
if len(weights) == 4:
hidden = hidden + weights[2][:, None, :]
y = F.gelu(hidden, approximate='none') @ w2
return y + weights[3][:, None, :] if len(weights) == 4 else y
def step(self, x, state: FastState, *, token_mask=None, timestep_mask=None,
differentiable=True, inner_trace=None):
"""Update all tokens once from incoming W; then read Q from updated W.
differentiable=False explicitly detaches incoming/outgoing state and
computes only inner gradients, even inside torch.no_grad(). An enclosing
torch.inference_mode() is rejected because inference tensors cannot take
part in the required autograd operation.
"""
if torch.is_inference_mode_enabled():
raise RuntimeError('TTT requires inner autograd; use no_grad, not inference_mode')
if x.ndim != 3 or x.shape[-1] != self.config.dim or x.device.type not in ('cpu', 'cuda'):
raise ValueError('Expected CPU/CUDA [B,S,D]')
b, s, _ = x.shape
if b <= 0 or s <= 0:
raise ValueError('Empty batch/token axis is unsupported; mask padding instead')
if state.updates.shape != (b,) or len(state.weights) != len(self.w0):
raise ValueError('Incompatible state')
for w, w0 in zip(state.weights, self.w0):
if w.shape != (b, *w0.shape) or w.dtype != x.dtype or w.device != x.device:
raise ValueError('State shape/dtype/device mismatch')
if x.dtype != self.alpha.dtype:
raise ValueError('Input/model dtype mismatch')
valid = torch.ones((b, s), dtype=torch.bool, device=x.device)
for mask, shape in ((token_mask, (b, s)), (timestep_mask, (b,))):
if mask is not None and (mask.shape != shape or mask.dtype != torch.bool):
raise ValueError('Masks must have bool token [B,S] / timestep [B] shapes')
if token_mask is not None:
valid = valid & token_mask
if timestep_mask is not None:
valid = valid & timestep_mask[:, None]
with torch.enable_grad():
# Include sanitizing in the explicit grad region so an enclosing
# no_grad does not silently cut the differentiable input branch.
clean = torch.where(valid[..., None], x, torch.zeros_like(x))
weights = tuple(w if w.requires_grad else w.detach().requires_grad_(True)
for w in (state.weights if differentiable else state.detach().weights))
# Local weight arguments isolate partial_W; clone preserves ALL outer paths.
weights = tuple(w.clone() for w in weights)
if differentiable:
q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr
else:
with torch.no_grad():
q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr
err = (self.fast(k, weights) - v).square()
per_trajectory = torch.where(valid[..., None], err, 0).sum((1, 2))
if self.config.reduction == 'mean':
per_trajectory = per_trajectory / (valid.sum(1) * self.config.memory_dim).clamp_min(1)
# Sum across independent trajectories; never divide by batch size.
grads = torch.autograd.grad(per_trajectory.sum(), weights,
create_graph=differentiable)
if inner_trace is not None:
inner_trace.update({'grads':[g.detach().cpu().clone() for g in grads],
'q':q.detach().cpu().clone(),'k':k.detach().cpu().clone(),'v':v.detach().cpu().clone()})
updated = tuple(w - lr * g for w, g in zip(weights, grads))
next_state = FastState(updated, state.updates + valid.any(1).long())
if differentiable:
memory = self.out(self.fast(q, updated))
y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0)
else:
with torch.no_grad():
memory = self.out(self.fast(q, updated))
y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0)
next_state = next_state.detach()
return y, next_state