"""State bridge for the unchanged pinned OpenTau TTT core.""" import torch from torch import nn from .core import FastState,TTTConfig from .reference_ttt import TTTMLPLayer,TTTFastWeights,TanhGate class ReferenceLayer(nn.Module): def __init__(self,width=1024): super().__init__() self.config=TTTConfig(dim=width,memory_dim=128,hidden_dim=256,base_lr=.1) self.core=TTTMLPLayer(width=width,num_heads=8,mlp_hidden_multiplier=2,base_lr=.1,rope_theta=10000.,scan_checkpoint_group_size=0,expected_mini_batch_size=26) self.gate=TanhGate(width,.001) @property def alpha(self):return self.gate.alpha @property def w0(self):return (self.core.w1_init,self.core.b1_init,self.core.w2_init,self.core.b2_init) @property def inner_lr(self):return self.alpha.new_tensor(self.core.base_lr) def initial_state(self,batch): w=self.core.initial_fast_weights(batch) return FastState(tuple(x.clone() for x in (w.w1,w.b1,w.w2,w.b2)),torch.zeros(batch,dtype=torch.long,device=self.alpha.device)) def step(self,x,state,*,differentiable): if not bool((state.updates==state.updates[0]).all()):raise NotImplementedError('Reference baseline uses synchronized B1 histories') weights=TTTFastWeights(*state.weights) out,w=self.core(x,mini_batch_size=x.shape[1],fast_weights=weights,position_offset=int(state.updates[0])*x.shape[1]) result=self.gate(x,out) next_state=FastState((w.w1,w.b1,w.w2,w.b2),state.updates+1) return result,next_state