File size: 1,541 Bytes
0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """State bridge for the unchanged pinned OpenTau TTT core."""
import torch
from torch import nn
from .core import FastState,TTTConfig
from .reference_ttt import TTTMLPLayer,TTTFastWeights,TanhGate
class ReferenceLayer(nn.Module):
def __init__(self,width=1024):
super().__init__()
self.config=TTTConfig(dim=width,memory_dim=128,hidden_dim=256,base_lr=.1)
self.core=TTTMLPLayer(width=width,num_heads=8,mlp_hidden_multiplier=2,base_lr=.1,rope_theta=10000.,scan_checkpoint_group_size=0,expected_mini_batch_size=26)
self.gate=TanhGate(width,.001)
@property
def alpha(self):return self.gate.alpha
@property
def w0(self):return (self.core.w1_init,self.core.b1_init,self.core.w2_init,self.core.b2_init)
@property
def inner_lr(self):return self.alpha.new_tensor(self.core.base_lr)
def initial_state(self,batch):
w=self.core.initial_fast_weights(batch)
return FastState(tuple(x.clone() for x in (w.w1,w.b1,w.w2,w.b2)),torch.zeros(batch,dtype=torch.long,device=self.alpha.device))
def step(self,x,state,*,differentiable):
if not bool((state.updates==state.updates[0]).all()):raise NotImplementedError('Reference baseline uses synchronized B1 histories')
weights=TTTFastWeights(*state.weights)
out,w=self.core(x,mini_batch_size=x.shape[1],fast_weights=weights,position_offset=int(state.updates[0])*x.shape[1])
result=self.gate(x,out)
next_state=FastState((w.w1,w.b1,w.w2,w.b2),state.updates+1)
return result,next_state
|