File size: 3,889 Bytes
add1d5f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import copy,unittest
import torch
from robottt_pi05.core import TTTLayer,TTTConfig
from robottt_pi05.layer_state import initial,reset,detach_all,clone,validate,parameter_storage_unique

torch.set_num_threads(2)

class CoreTests(unittest.TestCase):
    def setUp(self):torch.manual_seed(13)
    def test_lr_derivative_and_shared_forward(self):
        layer=TTTLayer(TTTConfig(dim=4,memory_dim=3,hidden_dim=6,base_lr=1e-4)).double()
        x=torch.randn(1,3,4,dtype=torch.float64)
        def objective():
            state=layer.initial_state(1)
            y,state=layer.step(x,state)
            y,state=layer.step(x+.2*y,state)
            return y.square().mean(),y,state
        loss,y,state=objective();g=torch.autograd.grad(loss,layer.raw_lr)[0]
        original=layer.raw_lr.detach().clone();h=.001
        with torch.no_grad():layer.raw_lr.copy_(original+h)
        plus=objective()[0].detach()
        with torch.no_grad():layer.raw_lr.copy_(original-h)
        minus=objective()[0].detach()
        with torch.no_grad():layer.raw_lr.copy_(original)
        fd=(plus-minus)/(2*h)
        torch.testing.assert_close(g,fd,atol=5e-11,rtol=2e-5)
        loss=objective()[0];params=[p for n,p in layer.named_parameters() if n!='raw_lr'];gs=torch.autograd.grad(loss,params)
        layer.raw_lr.requires_grad_(False);other,yp,st=objective();gf=torch.autograd.grad(other,params)
        self.assertTrue(torch.equal(y,yp));self.assertEqual(st.updates.tolist(),[2])
        for a,b in zip(state.weights,st.weights):self.assertTrue(torch.equal(a,b))
        for a,b in zip(gs,gf):self.assertTrue(torch.equal(a,b))
        with torch.no_grad():out,infer=layer.step(x,layer.initial_state(1),differentiable=False)
        self.assertEqual(infer.updates.tolist(),[1]);self.assertFalse(out.requires_grad)

    def test_storage_episode_reset_detach(self):
        modules=torch.nn.ModuleDict({str(i):TTTLayer(TTTConfig()).double() for i in [0,1]})
        self.assertTrue(parameter_storage_unique(modules));state=initial(modules,2);saved=clone(state,detach=True)
        copied=clone(state);copied['0'].weights[0][0].add_(1)
        self.assertTrue(torch.equal(state['0'].weights[0],saved['0'].weights[0]))
        self.assertTrue(torch.equal(copied['0'].weights[0][1],saved['0'].weights[0][1]))
        self.assertTrue(torch.equal(copied['1'].weights[0],saved['1'].weights[0]))
        detached=detach_all(modules,copied)
        self.assertFalse(any(w.requires_grad for s in detached.values() for w in s.weights))
        partial=reset(modules,detached,torch.tensor([True,False]),selected=['0'])
        self.assertTrue(torch.equal(partial['0'].weights[0][0],state['0'].weights[0][0]))
        self.assertTrue(torch.equal(partial['1'].weights[0],detached['1'].weights[0]))
        self.assertIs(reset(modules,detached,torch.tensor([False,False])),detached)
        with self.assertRaises(ValueError):validate(modules,{'0':state['0']})
        alias=dict(state);alias['1']=state['0']
        with self.assertRaises(ValueError):validate(modules,alias)

    def test_inner_local_partial_not_total(self):
        # Independent analytical fast-weight derivative with K depending on incoming U.
        layer=TTTLayer(TTTConfig(dim=2,memory_dim=2,hidden_dim=2,base_lr=1e-4)).double()
        state=layer.initial_state(1)
        x=state.weights[0].sum().reshape(1,1,1).expand(1,2,2)
        trace={};_,candidate=layer.step(x,state,inner_trace=trace)
        local=tuple(w.detach().clone().requires_grad_() for w in state.weights)
        binding=(layer.fast(layer.k(x).detach(),local)-layer.v(x).detach()).square().mean()
        reference=torch.autograd.grad(binding,local)
        for a,b in zip(trace['grads'],reference):torch.testing.assert_close(a,b,atol=1e-14,rtol=1e-12)
        self.assertTrue(any(w.grad_fn is not None for w in candidate.weights))

if __name__=='__main__':unittest.main()