File size: 3,617 Bytes
add1d5f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
"""Float64 proof that layer-first execution preserves multi-layer S8 TBPTT."""
import copy,unittest
import torch
from torch import nn
from robottt_pi05.core import TTTLayer,TTTConfig,FastState
try:
    from robottt_pi05.reference_core_adapter import ReferenceLayer
    from robottt_pi05.reference_ttt import TTTFastWeights
except ModuleNotFoundError as error:
    if error.name!='einops':raise
    ReferenceLayer=None

class LayerOrderTests(unittest.TestCase):
    def test_layer_first_matches_observation_first_loss_states_gradients(self):
        for kind in ('lite','reference'):
            if kind=='reference' and ReferenceLayer is None:continue
            torch.manual_seed(14)
            modules=nn.ModuleList([ReferenceLayer(16).double() if kind=='reference' else TTTLayer(TTTConfig(dim=16,memory_dim=4,hidden_dim=8,base_lr=1e-4)).double() for _ in range(2)])
            other=copy.deepcopy(modules);weights=[torch.randn(16,16,dtype=torch.float64)/8 for _ in range(4)]
            x=torch.randn(32,26,16,dtype=torch.float64);target=torch.randn_like(x)
            def execute(modules,layer_first):
                state={i:m.initial_state(1) for i,m in enumerate(modules)};losses=[];outputs=[]
                for lo in range(0,32,8):
                    hidden=x[lo:lo+8]
                    if layer_first:
                        for i,module in enumerate(modules):
                            branch=hidden@weights[2*i];old=state[i]
                            if kind=='reference':
                                y,w=module.core(branch.reshape(1,8*26,16),mini_batch_size=26,fast_weights=TTTFastWeights(*old.weights),position_offset=lo*26)
                                value=module.gate(branch,y.reshape_as(branch));state[i]=FastState((w.w1,w.b1,w.w2,w.b2),old.updates+8)
                            else:
                                values=[]
                                for t in range(8):
                                    y,state[i]=module.step(branch[t:t+1],state[i],differentiable=True);values.append(y)
                                value=torch.cat(values)
                            hidden=hidden+.7*value;hidden=hidden+torch.tanh(hidden@weights[2*i+1])
                    else:
                        values=[]
                        for t in range(8):
                            h=hidden[t:t+1]
                            for i,module in enumerate(modules):
                                branch=h@weights[2*i];value,state[i]=module.step(branch,state[i],differentiable=True)
                                h=h+.7*value;h=h+torch.tanh(h@weights[2*i+1])
                            values.append(h)
                        hidden=torch.cat(values)
                    loss=(hidden-target[lo:lo+8]).square().sum()/target.numel();loss.backward();losses.append(float(loss.detach()));outputs.append(hidden.detach())
                    state={i:s.detach() for i,s in state.items()}
                return sum(losses),torch.cat(outputs),state
            a=execute(modules,False);b=execute(other,True)
            self.assertAlmostEqual(a[0],b[0],places=12)
            torch.testing.assert_close(a[1],b[1],rtol=1e-11,atol=1e-12)
            for i in range(2):
                self.assertEqual(b[2][i].updates.tolist(),[32])
                for aw,bw in zip(a[2][i].weights,b[2][i].weights):torch.testing.assert_close(aw,bw,rtol=1e-10,atol=1e-12)
            for ap,bp in zip(modules.parameters(),other.parameters()):
                self.assertIsNotNone(ap.grad);self.assertIsNotNone(bp.grad)
                torch.testing.assert_close(ap.grad,bp.grad,rtol=1e-9,atol=1e-11)