Download code/tests/test_layer_order.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 3.62 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/tests/test_layer_order.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/tests/test_layer_order.py
-
curl -L -o test_layer_order.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/tests/test_layer_order.py
3.62 kB
| """Float64 proof that layer-first execution preserves multi-layer S8 TBPTT.""" | |
| import copy,unittest | |
| import torch | |
| from torch import nn | |
| from robottt_pi05.core import TTTLayer,TTTConfig,FastState | |
| try: | |
| from robottt_pi05.reference_core_adapter import ReferenceLayer | |
| from robottt_pi05.reference_ttt import TTTFastWeights | |
| except ModuleNotFoundError as error: | |
| if error.name!='einops':raise | |
| ReferenceLayer=None | |
| class LayerOrderTests(unittest.TestCase): | |
| def test_layer_first_matches_observation_first_loss_states_gradients(self): | |
| for kind in ('lite','reference'): | |
| if kind=='reference' and ReferenceLayer is None:continue | |
| torch.manual_seed(14) | |
| modules=nn.ModuleList([ReferenceLayer(16).double() if kind=='reference' else TTTLayer(TTTConfig(dim=16,memory_dim=4,hidden_dim=8,base_lr=1e-4)).double() for _ in range(2)]) | |
| other=copy.deepcopy(modules);weights=[torch.randn(16,16,dtype=torch.float64)/8 for _ in range(4)] | |
| x=torch.randn(32,26,16,dtype=torch.float64);target=torch.randn_like(x) | |
| def execute(modules,layer_first): | |
| state={i:m.initial_state(1) for i,m in enumerate(modules)};losses=[];outputs=[] | |
| for lo in range(0,32,8): | |
| hidden=x[lo:lo+8] | |
| if layer_first: | |
| for i,module in enumerate(modules): | |
| branch=hidden@weights[2*i];old=state[i] | |
| if kind=='reference': | |
| y,w=module.core(branch.reshape(1,8*26,16),mini_batch_size=26,fast_weights=TTTFastWeights(*old.weights),position_offset=lo*26) | |
| value=module.gate(branch,y.reshape_as(branch));state[i]=FastState((w.w1,w.b1,w.w2,w.b2),old.updates+8) | |
| else: | |
| values=[] | |
| for t in range(8): | |
| y,state[i]=module.step(branch[t:t+1],state[i],differentiable=True);values.append(y) | |
| value=torch.cat(values) | |
| hidden=hidden+.7*value;hidden=hidden+torch.tanh(hidden@weights[2*i+1]) | |
| else: | |
| values=[] | |
| for t in range(8): | |
| h=hidden[t:t+1] | |
| for i,module in enumerate(modules): | |
| branch=h@weights[2*i];value,state[i]=module.step(branch,state[i],differentiable=True) | |
| h=h+.7*value;h=h+torch.tanh(h@weights[2*i+1]) | |
| values.append(h) | |
| hidden=torch.cat(values) | |
| loss=(hidden-target[lo:lo+8]).square().sum()/target.numel();loss.backward();losses.append(float(loss.detach()));outputs.append(hidden.detach()) | |
| state={i:s.detach() for i,s in state.items()} | |
| return sum(losses),torch.cat(outputs),state | |
| a=execute(modules,False);b=execute(other,True) | |
| self.assertAlmostEqual(a[0],b[0],places=12) | |
| torch.testing.assert_close(a[1],b[1],rtol=1e-11,atol=1e-12) | |
| for i in range(2): | |
| self.assertEqual(b[2][i].updates.tolist(),[32]) | |
| for aw,bw in zip(a[2][i].weights,b[2][i].weights):torch.testing.assert_close(aw,bw,rtol=1e-10,atol=1e-12) | |
| for ap,bp in zip(modules.parameters(),other.parameters()): | |
| self.assertIsNotNone(ap.grad);self.assertIsNotNone(bp.grad) | |
| torch.testing.assert_close(ap.grad,bp.grad,rtol=1e-9,atol=1e-11) | |