File size: 3,889 Bytes
add1d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | import copy,unittest
import torch
from robottt_pi05.core import TTTLayer,TTTConfig
from robottt_pi05.layer_state import initial,reset,detach_all,clone,validate,parameter_storage_unique
torch.set_num_threads(2)
class CoreTests(unittest.TestCase):
def setUp(self):torch.manual_seed(13)
def test_lr_derivative_and_shared_forward(self):
layer=TTTLayer(TTTConfig(dim=4,memory_dim=3,hidden_dim=6,base_lr=1e-4)).double()
x=torch.randn(1,3,4,dtype=torch.float64)
def objective():
state=layer.initial_state(1)
y,state=layer.step(x,state)
y,state=layer.step(x+.2*y,state)
return y.square().mean(),y,state
loss,y,state=objective();g=torch.autograd.grad(loss,layer.raw_lr)[0]
original=layer.raw_lr.detach().clone();h=.001
with torch.no_grad():layer.raw_lr.copy_(original+h)
plus=objective()[0].detach()
with torch.no_grad():layer.raw_lr.copy_(original-h)
minus=objective()[0].detach()
with torch.no_grad():layer.raw_lr.copy_(original)
fd=(plus-minus)/(2*h)
torch.testing.assert_close(g,fd,atol=5e-11,rtol=2e-5)
loss=objective()[0];params=[p for n,p in layer.named_parameters() if n!='raw_lr'];gs=torch.autograd.grad(loss,params)
layer.raw_lr.requires_grad_(False);other,yp,st=objective();gf=torch.autograd.grad(other,params)
self.assertTrue(torch.equal(y,yp));self.assertEqual(st.updates.tolist(),[2])
for a,b in zip(state.weights,st.weights):self.assertTrue(torch.equal(a,b))
for a,b in zip(gs,gf):self.assertTrue(torch.equal(a,b))
with torch.no_grad():out,infer=layer.step(x,layer.initial_state(1),differentiable=False)
self.assertEqual(infer.updates.tolist(),[1]);self.assertFalse(out.requires_grad)
def test_storage_episode_reset_detach(self):
modules=torch.nn.ModuleDict({str(i):TTTLayer(TTTConfig()).double() for i in [0,1]})
self.assertTrue(parameter_storage_unique(modules));state=initial(modules,2);saved=clone(state,detach=True)
copied=clone(state);copied['0'].weights[0][0].add_(1)
self.assertTrue(torch.equal(state['0'].weights[0],saved['0'].weights[0]))
self.assertTrue(torch.equal(copied['0'].weights[0][1],saved['0'].weights[0][1]))
self.assertTrue(torch.equal(copied['1'].weights[0],saved['1'].weights[0]))
detached=detach_all(modules,copied)
self.assertFalse(any(w.requires_grad for s in detached.values() for w in s.weights))
partial=reset(modules,detached,torch.tensor([True,False]),selected=['0'])
self.assertTrue(torch.equal(partial['0'].weights[0][0],state['0'].weights[0][0]))
self.assertTrue(torch.equal(partial['1'].weights[0],detached['1'].weights[0]))
self.assertIs(reset(modules,detached,torch.tensor([False,False])),detached)
with self.assertRaises(ValueError):validate(modules,{'0':state['0']})
alias=dict(state);alias['1']=state['0']
with self.assertRaises(ValueError):validate(modules,alias)
def test_inner_local_partial_not_total(self):
# Independent analytical fast-weight derivative with K depending on incoming U.
layer=TTTLayer(TTTConfig(dim=2,memory_dim=2,hidden_dim=2,base_lr=1e-4)).double()
state=layer.initial_state(1)
x=state.weights[0].sum().reshape(1,1,1).expand(1,2,2)
trace={};_,candidate=layer.step(x,state,inner_trace=trace)
local=tuple(w.detach().clone().requires_grad_() for w in state.weights)
binding=(layer.fast(layer.k(x).detach(),local)-layer.v(x).detach()).square().mean()
reference=torch.autograd.grad(binding,local)
for a,b in zip(trace['grads'],reference):torch.testing.assert_close(a,b,atol=1e-14,rtol=1e-12)
self.assertTrue(any(w.grad_fn is not None for w in candidate.weights))
if __name__=='__main__':unittest.main()
|