Download code/src/robottt_pi05/core.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 8.3 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/core.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/src/robottt_pi05/core.py
-
curl -L -o core.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/core.py
8.3 kB
| """Reviewed reference copy; CPU/CUDA input guard only changed for integration.""" | |
| from dataclasses import dataclass | |
| import math | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class TTTConfig: | |
| dim: int = 4 | |
| memory_dim: int = 3 | |
| hidden_dim: int = 6 | |
| bias: bool = True | |
| reduction: str = 'mean' # per-trajectory token/feature mean, or sum | |
| base_lr: float = 0.1 | |
| lr_parameterization: str = 'softplus' # or exp; multiplier starts at one | |
| output_projection: bool = True | |
| def __post_init__(self): | |
| if min(self.dim, self.memory_dim, self.hidden_dim) <= 0: | |
| raise ValueError('Dimensions must be positive') | |
| if self.reduction not in ('mean', 'sum'): | |
| raise ValueError('reduction must be mean or sum') | |
| if self.lr_parameterization not in ('softplus', 'exp') or self.base_lr <= 0: | |
| raise ValueError('Invalid learning rate configuration') | |
| if not self.output_projection and self.dim != self.memory_dim: | |
| raise ValueError('Without output projection, dim must equal memory_dim') | |
| class FastState: | |
| weights: tuple[torch.Tensor, ...] | |
| updates: torch.Tensor # [B], valid observation updates, not token positions | |
| def detach(self): | |
| # Detach the complete W, NOT a delta followed by adding W0. | |
| return FastState(tuple(w.detach() for w in self.weights), self.updates.detach()) | |
| class TTTLayer(nn.Module): | |
| def __init__(self, config: TTTConfig): | |
| super().__init__() | |
| self.config = config | |
| d, r, h = config.dim, config.memory_dim, config.hidden_dim | |
| self.q = nn.Linear(d, r, bias=False) | |
| self.k = nn.Linear(d, r, bias=False) | |
| self.v = nn.Linear(d, r, bias=False) | |
| self.out = nn.Linear(r, d, bias=False) if config.output_projection else nn.Identity() | |
| w1, w2 = torch.empty(r, h), torch.empty(h, r) | |
| nn.init.xavier_uniform_(w1) | |
| nn.init.xavier_uniform_(w2) | |
| values = [w1, w2] | |
| if config.bias: | |
| values += [torch.zeros(h), torch.zeros(r)] | |
| self.w0 = nn.ParameterList([nn.Parameter(w) for w in values]) | |
| init = math.log(math.expm1(1.0)) if config.lr_parameterization == 'softplus' else 0.0 | |
| self.raw_lr = nn.Parameter(torch.tensor(init)) | |
| self.alpha = nn.Parameter(torch.full((d,), 0.001)) | |
| def inner_lr(self): | |
| multiplier = F.softplus(self.raw_lr) if self.config.lr_parameterization == 'softplus' else self.raw_lr.exp() | |
| return self.config.base_lr * multiplier | |
| def forward(self, x, state, **kwargs): | |
| return self.step(x, state, **kwargs) | |
| def initial_state(self, batch_size: int): | |
| if batch_size <= 0: | |
| raise ValueError('batch_size must be positive') | |
| # Clone gives independent storage while preserving the path to learned W0. | |
| return FastState(tuple(w.unsqueeze(0).expand(batch_size, *w.shape).clone() | |
| for w in self.w0), | |
| torch.zeros(batch_size, dtype=torch.long, device=self.alpha.device)) | |
| def reset(self, state: FastState, reset_mask: torch.Tensor): | |
| b = state.updates.shape[0] | |
| if reset_mask.shape != (b,) or reset_mask.dtype != torch.bool: | |
| raise ValueError('reset_mask must be bool [B]') | |
| # A no-op reset must not attach W0 with a zero gradient: optimizers can | |
| # still apply momentum/weight decay when grad is zero rather than None. | |
| if not bool(reset_mask.any()): | |
| return state | |
| fresh = self.initial_state(b) | |
| return FastState(tuple(torch.where(reset_mask.reshape(b, *([1] * (w.ndim-1))), w0, w) | |
| for w, w0 in zip(state.weights, fresh.weights)), | |
| torch.where(reset_mask, fresh.updates, state.updates)) | |
| def fast(z, weights): | |
| w1, w2 = weights[:2] | |
| hidden = z @ w1 | |
| if len(weights) == 4: | |
| hidden = hidden + weights[2][:, None, :] | |
| y = F.gelu(hidden, approximate='none') @ w2 | |
| return y + weights[3][:, None, :] if len(weights) == 4 else y | |
| def step(self, x, state: FastState, *, token_mask=None, timestep_mask=None, | |
| differentiable=True, inner_trace=None): | |
| """Update all tokens once from incoming W; then read Q from updated W. | |
| differentiable=False explicitly detaches incoming/outgoing state and | |
| computes only inner gradients, even inside torch.no_grad(). An enclosing | |
| torch.inference_mode() is rejected because inference tensors cannot take | |
| part in the required autograd operation. | |
| """ | |
| if torch.is_inference_mode_enabled(): | |
| raise RuntimeError('TTT requires inner autograd; use no_grad, not inference_mode') | |
| if x.ndim != 3 or x.shape[-1] != self.config.dim or x.device.type not in ('cpu', 'cuda'): | |
| raise ValueError('Expected CPU/CUDA [B,S,D]') | |
| b, s, _ = x.shape | |
| if b <= 0 or s <= 0: | |
| raise ValueError('Empty batch/token axis is unsupported; mask padding instead') | |
| if state.updates.shape != (b,) or len(state.weights) != len(self.w0): | |
| raise ValueError('Incompatible state') | |
| for w, w0 in zip(state.weights, self.w0): | |
| if w.shape != (b, *w0.shape) or w.dtype != x.dtype or w.device != x.device: | |
| raise ValueError('State shape/dtype/device mismatch') | |
| if x.dtype != self.alpha.dtype: | |
| raise ValueError('Input/model dtype mismatch') | |
| valid = torch.ones((b, s), dtype=torch.bool, device=x.device) | |
| for mask, shape in ((token_mask, (b, s)), (timestep_mask, (b,))): | |
| if mask is not None and (mask.shape != shape or mask.dtype != torch.bool): | |
| raise ValueError('Masks must have bool token [B,S] / timestep [B] shapes') | |
| if token_mask is not None: | |
| valid = valid & token_mask | |
| if timestep_mask is not None: | |
| valid = valid & timestep_mask[:, None] | |
| with torch.enable_grad(): | |
| # Include sanitizing in the explicit grad region so an enclosing | |
| # no_grad does not silently cut the differentiable input branch. | |
| clean = torch.where(valid[..., None], x, torch.zeros_like(x)) | |
| weights = tuple(w if w.requires_grad else w.detach().requires_grad_(True) | |
| for w in (state.weights if differentiable else state.detach().weights)) | |
| # Local weight arguments isolate partial_W; clone preserves ALL outer paths. | |
| weights = tuple(w.clone() for w in weights) | |
| if differentiable: | |
| q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr | |
| else: | |
| with torch.no_grad(): | |
| q, k, v, lr = self.q(clean), self.k(clean), self.v(clean), self.inner_lr | |
| err = (self.fast(k, weights) - v).square() | |
| per_trajectory = torch.where(valid[..., None], err, 0).sum((1, 2)) | |
| if self.config.reduction == 'mean': | |
| per_trajectory = per_trajectory / (valid.sum(1) * self.config.memory_dim).clamp_min(1) | |
| # Sum across independent trajectories; never divide by batch size. | |
| grads = torch.autograd.grad(per_trajectory.sum(), weights, | |
| create_graph=differentiable) | |
| if inner_trace is not None: | |
| inner_trace.update({'grads':[g.detach().cpu().clone() for g in grads], | |
| 'q':q.detach().cpu().clone(),'k':k.detach().cpu().clone(),'v':v.detach().cpu().clone()}) | |
| updated = tuple(w - lr * g for w, g in zip(weights, grads)) | |
| next_state = FastState(updated, state.updates + valid.any(1).long()) | |
| if differentiable: | |
| memory = self.out(self.fast(q, updated)) | |
| y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0) | |
| else: | |
| with torch.no_grad(): | |
| memory = self.out(self.fast(q, updated)) | |
| y = x + torch.tanh(self.alpha) * torch.where(valid[..., None], memory, 0) | |
| next_state = next_state.detach() | |
| return y, next_state | |