Download code/src/robottt_pi05/reference_core_adapter.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 1.54 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/reference_core_adapter.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/src/robottt_pi05/reference_core_adapter.py
-
curl -L -o reference_core_adapter.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/reference_core_adapter.py
1.54 kB
| """State bridge for the unchanged pinned OpenTau TTT core.""" | |
| import torch | |
| from torch import nn | |
| from .core import FastState,TTTConfig | |
| from .reference_ttt import TTTMLPLayer,TTTFastWeights,TanhGate | |
| class ReferenceLayer(nn.Module): | |
| def __init__(self,width=1024): | |
| super().__init__() | |
| self.config=TTTConfig(dim=width,memory_dim=128,hidden_dim=256,base_lr=.1) | |
| self.core=TTTMLPLayer(width=width,num_heads=8,mlp_hidden_multiplier=2,base_lr=.1,rope_theta=10000.,scan_checkpoint_group_size=0,expected_mini_batch_size=26) | |
| self.gate=TanhGate(width,.001) | |
| def alpha(self):return self.gate.alpha | |
| def w0(self):return (self.core.w1_init,self.core.b1_init,self.core.w2_init,self.core.b2_init) | |
| def inner_lr(self):return self.alpha.new_tensor(self.core.base_lr) | |
| def initial_state(self,batch): | |
| w=self.core.initial_fast_weights(batch) | |
| return FastState(tuple(x.clone() for x in (w.w1,w.b1,w.w2,w.b2)),torch.zeros(batch,dtype=torch.long,device=self.alpha.device)) | |
| def step(self,x,state,*,differentiable): | |
| if not bool((state.updates==state.updates[0]).all()):raise NotImplementedError('Reference baseline uses synchronized B1 histories') | |
| weights=TTTFastWeights(*state.weights) | |
| out,w=self.core(x,mini_batch_size=x.shape[1],fast_weights=weights,position_offset=int(state.updates[0])*x.shape[1]) | |
| result=self.gate(x,out) | |
| next_state=FastState((w.w1,w.b1,w.w2,w.b2),state.updates+1) | |
| return result,next_state | |