File size: 3,617 Bytes
add1d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Float64 proof that layer-first execution preserves multi-layer S8 TBPTT."""
import copy,unittest
import torch
from torch import nn
from robottt_pi05.core import TTTLayer,TTTConfig,FastState
try:
from robottt_pi05.reference_core_adapter import ReferenceLayer
from robottt_pi05.reference_ttt import TTTFastWeights
except ModuleNotFoundError as error:
if error.name!='einops':raise
ReferenceLayer=None
class LayerOrderTests(unittest.TestCase):
def test_layer_first_matches_observation_first_loss_states_gradients(self):
for kind in ('lite','reference'):
if kind=='reference' and ReferenceLayer is None:continue
torch.manual_seed(14)
modules=nn.ModuleList([ReferenceLayer(16).double() if kind=='reference' else TTTLayer(TTTConfig(dim=16,memory_dim=4,hidden_dim=8,base_lr=1e-4)).double() for _ in range(2)])
other=copy.deepcopy(modules);weights=[torch.randn(16,16,dtype=torch.float64)/8 for _ in range(4)]
x=torch.randn(32,26,16,dtype=torch.float64);target=torch.randn_like(x)
def execute(modules,layer_first):
state={i:m.initial_state(1) for i,m in enumerate(modules)};losses=[];outputs=[]
for lo in range(0,32,8):
hidden=x[lo:lo+8]
if layer_first:
for i,module in enumerate(modules):
branch=hidden@weights[2*i];old=state[i]
if kind=='reference':
y,w=module.core(branch.reshape(1,8*26,16),mini_batch_size=26,fast_weights=TTTFastWeights(*old.weights),position_offset=lo*26)
value=module.gate(branch,y.reshape_as(branch));state[i]=FastState((w.w1,w.b1,w.w2,w.b2),old.updates+8)
else:
values=[]
for t in range(8):
y,state[i]=module.step(branch[t:t+1],state[i],differentiable=True);values.append(y)
value=torch.cat(values)
hidden=hidden+.7*value;hidden=hidden+torch.tanh(hidden@weights[2*i+1])
else:
values=[]
for t in range(8):
h=hidden[t:t+1]
for i,module in enumerate(modules):
branch=h@weights[2*i];value,state[i]=module.step(branch,state[i],differentiable=True)
h=h+.7*value;h=h+torch.tanh(h@weights[2*i+1])
values.append(h)
hidden=torch.cat(values)
loss=(hidden-target[lo:lo+8]).square().sum()/target.numel();loss.backward();losses.append(float(loss.detach()));outputs.append(hidden.detach())
state={i:s.detach() for i,s in state.items()}
return sum(losses),torch.cat(outputs),state
a=execute(modules,False);b=execute(other,True)
self.assertAlmostEqual(a[0],b[0],places=12)
torch.testing.assert_close(a[1],b[1],rtol=1e-11,atol=1e-12)
for i in range(2):
self.assertEqual(b[2][i].updates.tolist(),[32])
for aw,bw in zip(a[2][i].weights,b[2][i].weights):torch.testing.assert_close(aw,bw,rtol=1e-10,atol=1e-12)
for ap,bp in zip(modules.parameters(),other.parameters()):
self.assertIsNotNone(ap.grad);self.assertIsNotNone(bp.grad)
torch.testing.assert_close(ap.grad,bp.grad,rtol=1e-9,atol=1e-11)
|