File size: 1,912 Bytes
add1d5f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
"""Parity of the state bridge against the unchanged OpenTau scan."""
import copy,unittest
import torch
try:
    from robottt_pi05.reference_core_adapter import ReferenceLayer
except ModuleNotFoundError as error:
    if error.name!='einops':raise
    ReferenceLayer=None
from robottt_pi05.optimization import learning_rate

@unittest.skipIf(ReferenceLayer is None,'Install the optional reference extra')
class ReferenceParityTests(unittest.TestCase):
    def test_sequential_bridge_equals_native_scan_and_gradients(self):
        torch.manual_seed(9);wrapped=ReferenceLayer(width=16).double();native=copy.deepcopy(wrapped)
        x=torch.randn(1,3*26,16,dtype=torch.float64)
        state=wrapped.initial_state(1);outputs=[]
        for t in range(3):
            y,state=wrapped.step(x[:,t*26:(t+1)*26],state,differentiable=True);outputs.append(y)
        sequence=torch.cat(outputs,1)
        y,fast=native.core(x,mini_batch_size=26)
        expected=native.gate(x,y)
        torch.testing.assert_close(sequence,expected,rtol=1e-12,atol=1e-12)
        for a,b in zip(state.weights,(fast.w1,fast.b1,fast.w2,fast.b2)):torch.testing.assert_close(a,b,rtol=1e-12,atol=1e-12)
        sequence.square().mean().backward();expected.square().mean().backward()
        for (an,ap),(bn,bp) in zip(wrapped.named_parameters(),native.named_parameters()):
            self.assertEqual(an,bn);self.assertIsNotNone(ap.grad)
            torch.testing.assert_close(ap.grad,bp.grad,rtol=1e-9,atol=1e-11)
        self.assertEqual(state.updates.tolist(),[3])

    def test_explicit_schedule(self):
        self.assertAlmostEqual(learning_rate(0,'opentau_3000'),1e-4/101)
        self.assertAlmostEqual(learning_rate(3000,'opentau_3000'),1e-5)
        self.assertGreater(learning_rate(99,'opentau_3000'),learning_rate(0,'opentau_3000'))
        self.assertEqual(learning_rate(10000,'opentau_3000'),learning_rate(3000,'opentau_3000'))