"""Float64 proof that layer-first execution preserves multi-layer S8 TBPTT.""" import copy,unittest import torch from torch import nn from robottt_pi05.core import TTTLayer,TTTConfig,FastState try: from robottt_pi05.reference_core_adapter import ReferenceLayer from robottt_pi05.reference_ttt import TTTFastWeights except ModuleNotFoundError as error: if error.name!='einops':raise ReferenceLayer=None class LayerOrderTests(unittest.TestCase): def test_layer_first_matches_observation_first_loss_states_gradients(self): for kind in ('lite','reference'): if kind=='reference' and ReferenceLayer is None:continue torch.manual_seed(14) modules=nn.ModuleList([ReferenceLayer(16).double() if kind=='reference' else TTTLayer(TTTConfig(dim=16,memory_dim=4,hidden_dim=8,base_lr=1e-4)).double() for _ in range(2)]) other=copy.deepcopy(modules);weights=[torch.randn(16,16,dtype=torch.float64)/8 for _ in range(4)] x=torch.randn(32,26,16,dtype=torch.float64);target=torch.randn_like(x) def execute(modules,layer_first): state={i:m.initial_state(1) for i,m in enumerate(modules)};losses=[];outputs=[] for lo in range(0,32,8): hidden=x[lo:lo+8] if layer_first: for i,module in enumerate(modules): branch=hidden@weights[2*i];old=state[i] if kind=='reference': y,w=module.core(branch.reshape(1,8*26,16),mini_batch_size=26,fast_weights=TTTFastWeights(*old.weights),position_offset=lo*26) value=module.gate(branch,y.reshape_as(branch));state[i]=FastState((w.w1,w.b1,w.w2,w.b2),old.updates+8) else: values=[] for t in range(8): y,state[i]=module.step(branch[t:t+1],state[i],differentiable=True);values.append(y) value=torch.cat(values) hidden=hidden+.7*value;hidden=hidden+torch.tanh(hidden@weights[2*i+1]) else: values=[] for t in range(8): h=hidden[t:t+1] for i,module in enumerate(modules): branch=h@weights[2*i];value,state[i]=module.step(branch,state[i],differentiable=True) h=h+.7*value;h=h+torch.tanh(h@weights[2*i+1]) values.append(h) hidden=torch.cat(values) loss=(hidden-target[lo:lo+8]).square().sum()/target.numel();loss.backward();losses.append(float(loss.detach()));outputs.append(hidden.detach()) state={i:s.detach() for i,s in state.items()} return sum(losses),torch.cat(outputs),state a=execute(modules,False);b=execute(other,True) self.assertAlmostEqual(a[0],b[0],places=12) torch.testing.assert_close(a[1],b[1],rtol=1e-11,atol=1e-12) for i in range(2): self.assertEqual(b[2][i].updates.tolist(),[32]) for aw,bw in zip(a[2][i].weights,b[2][i].weights):torch.testing.assert_close(aw,bw,rtol=1e-10,atol=1e-12) for ap,bp in zip(modules.parameters(),other.parameters()): self.assertIsNotNone(ap.grad);self.assertIsNotNone(bp.grad) torch.testing.assert_close(ap.grad,bp.grad,rtol=1e-9,atol=1e-11)