File size: 1,541 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
"""State bridge for the unchanged pinned OpenTau TTT core."""
import torch
from torch import nn
from .core import FastState,TTTConfig
from .reference_ttt import TTTMLPLayer,TTTFastWeights,TanhGate

class ReferenceLayer(nn.Module):
    def __init__(self,width=1024):
        super().__init__()
        self.config=TTTConfig(dim=width,memory_dim=128,hidden_dim=256,base_lr=.1)
        self.core=TTTMLPLayer(width=width,num_heads=8,mlp_hidden_multiplier=2,base_lr=.1,rope_theta=10000.,scan_checkpoint_group_size=0,expected_mini_batch_size=26)
        self.gate=TanhGate(width,.001)
    @property
    def alpha(self):return self.gate.alpha
    @property
    def w0(self):return (self.core.w1_init,self.core.b1_init,self.core.w2_init,self.core.b2_init)
    @property
    def inner_lr(self):return self.alpha.new_tensor(self.core.base_lr)
    def initial_state(self,batch):
        w=self.core.initial_fast_weights(batch)
        return FastState(tuple(x.clone() for x in (w.w1,w.b1,w.w2,w.b2)),torch.zeros(batch,dtype=torch.long,device=self.alpha.device))
    def step(self,x,state,*,differentiable):
        if not bool((state.updates==state.updates[0]).all()):raise NotImplementedError('Reference baseline uses synchronized B1 histories')
        weights=TTTFastWeights(*state.weights)
        out,w=self.core(x,mini_batch_size=x.shape[1],fast_weights=weights,position_offset=int(state.updates[0])*x.shape[1])
        result=self.gate(x,out)
        next_state=FastState((w.w1,w.b1,w.w2,w.b2),state.updates+1)
        return result,next_state