File size: 10,900 Bytes
fb0074a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
"""Non-multiple-of-32 sessions must not disable native input integrity checks."""
import tempfile
import unittest
from unittest.mock import patch
import json
from pathlib import Path
import numpy as np
from common import WORK, OMG, OMNI, EVALUATOR, sha256, write_json
from native_eval import install_tail_policy, install_bounded_motion_conversion


class EntryPointTests(unittest.TestCase):
    def test_provenance_does_not_prepopulate_native_output(self):
        import native_eval
        with tempfile.TemporaryDirectory() as tmp:
            output=Path(tmp)/'new_output'

            def native_main(*args, **kwargs):
                self.assertFalse(output.exists())
                output.mkdir()
                raise SystemExit(0)

            with patch.object(native_eval.sys,'argv',['native_eval.py','--output-dir',str(output)]), \
                 patch.object(native_eval,'install_tail_policy'), \
                 patch.object(native_eval,'install_bounded_motion_conversion'), \
                 patch.object(native_eval.runpy,'run_path',side_effect=native_main):
                native_eval.main()
            self.assertTrue((output/'hfd_adapter.json').is_file())


class BoundedFKTests(unittest.TestCase):
    def test_native_fk_matches_for_all_sources_frames_and_motion_keys(self):
        import torch
        from omg.benchmarks.runners import common
        from omg.robots.g1.kinematics import G1Kinematics
        kin=G1Kinematics(str(OMG/'assets/robots/g1/g1_kinematics.json')).eval()
        rng=torch.Generator().manual_seed(42)
        qpos=torch.randn((33,9,36),generator=rng)*.1
        qpos[...,2]+=.8
        quat=torch.randn((33,9,4),generator=rng)
        qpos[...,3:7]=quat/torch.linalg.vector_norm(quat,dim=-1,keepdim=True)
        original=common._motion_for_evaluator
        try:
            install_bounded_motion_conversion()
            with torch.inference_mode():
                for key in ('qpos_36','body_pos_local','body_link_pos_local'):
                    expected=original(qpos,key,kinematics=kin)
                    actual=common._motion_for_evaluator(qpos,key,kinematics=kin)
                    torch.testing.assert_close(actual,expected,rtol=0,atol=0)
        finally:
            common._motion_for_evaluator=original


class ScoreBundleTests(unittest.TestCase):
    def fixture(self, root):
        import importlib.util
        from metrics import build_paper_payload
        from session_data import flattened_rows
        spec=importlib.util.spec_from_file_location('hfd_score_check',WORK/'evaluate.py')
        module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
        cohort,generated,output=(root/x for x in ('cohort','generated','eval'))
        native=output/'motion';native.mkdir(parents=True)
        original=[dict(sample_id=f's{i}',caption='stand',reference_frames=51) for i in range(32)]
        write_json(cohort/'manifest.json',dict(rows=original))
        rows=flattened_rows(original);ids=np.array([r['sample_id'] for r in rows])
        identity=dict(step=1,cohort_sha256=sha256(cohort/'manifest.json'))
        write_json(generated/'identity.json',identity)
        write_json(output/'protocol.json',dict(**identity,count=32,session_count=32,
            source_hashes={str(p):sha256(p) for p in (OMNI/'evaluate.py',OMNI/'metrics.py',EVALUATOR)}))
        write_json(native/'run_state.json',dict(status='complete',kind='benchmark',paper_row_complete=True))
        raw=dict(protocol=dict(num_samples=32),generated_target=dict(
            text=dict(motion_fid=.1,matching_score=1.,r_precision=[.3,.5,.7],diversity_generated=1.2,
                generated_retrieval_diagnostics=dict(retrieval_num_samples=32),
                reference_retrieval_diagnostics=dict(retrieval_num_samples=32)),
            physical=dict(contact_sliding_speed=dict(mean=.1),body_jerk_mean=dict(mean=20.))),
            tracker_executed=dict(enabled=True,paper_eligible=True,num_samples=32,
                metrics={k:dict(mean=1.) for k in ('mpjpe','g_mpjpe','e_vel','e_acc')},
                failures=dict(fall_rate_percent=0.,joint_limit_rate_percent=0.)))
        write_json(native/'metrics.json',raw)
        write_json(native/'paper_metrics.json',build_paper_payload('text',raw))
        np.savez(native/'sample_metrics.npz',sample_id=ids,
                 contact_sliding_speed=np.full(32,.1),body_jerk_mean=np.full(32,20.))
        np.savez(native/'tracker_failure_flags.npz',sample_id=ids,
                 **{k:np.ones(32) for k in ('mpjpe','g_mpjpe','e_vel','e_acc')},
                 fall=np.zeros(32,bool),joint_limit=np.zeros(32,bool))
        np.savez(native/'embeddings.npz',sample_id=ids,
                 **{k:np.ones((32,512),np.float32) for k in ('reference_embeddings','generated_embeddings',
                     'text_embeddings','distribution_reference_embeddings','distribution_generated_embeddings')})
        (output/'inputs').mkdir()
        np.savez(output/'inputs/conditions.npz',sample_id=ids,captions=np.array(['stand']*32))
        write_json(output/'phase_physics.json',dict(summary=dict(all=dict(sources=32)),
            per_segment=[dict(sample_id=r['sample_id'],phase='all',frames=31,
                              generated=dict(jerk=1.),gt=dict(jerk=1.)) for r in rows]))
        write_json(native/'hfd_adapter.json',dict(adapter_sha256=sha256(WORK/'native_eval.py')))
        return module.verify_scores,(cohort,generated,output)

    def test_complete_bundle_and_incomplete_status(self):
        with tempfile.TemporaryDirectory() as tmp:
            verify,args=self.fixture(Path(tmp));out=args[-1]
            self.assertTrue(verify(*args)['complete'])
            state=out/'motion/run_state.json'
            write_json(state,dict(status='running',kind='benchmark',paper_row_complete=True))
            with self.assertRaisesRegex(ValueError,'incomplete'):verify(*args)

    def test_mixed_order_nonfinite_and_inconsistent_table_rejected(self):
        with tempfile.TemporaryDirectory() as tmp:
            verify,args=self.fixture(Path(tmp));out=args[-1];native=out/'motion'
            path=native/'embeddings.npz'
            with np.load(path) as z:original={k:z[k] for k in z.files}
            np.savez(path,**{**original,'sample_id':original['sample_id'][::-1]})
            with self.assertRaisesRegex(ValueError,'identities/order'):verify(*args)
            values=original['generated_embeddings'].copy();values[-1]=np.nan
            np.savez(path,**{**original,'generated_embeddings':values})
            with self.assertRaisesRegex(ValueError,'invalid array'):verify(*args)
            np.savez(path,**original)
            paper=json.loads((native/'paper_metrics.json').read_text());paper['values'][0]+=1
            write_json(native/'paper_metrics.json',paper)
            with self.assertRaisesRegex(ValueError,'Displayed metrics'):verify(*args)


class TailPolicyTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        import metrics
        from omg.benchmarks.runners import text
        cls.metrics,cls.text=metrics,text
        cls.original_loader=staticmethod(metrics.load_benchmark_inputs)
        cls.original_summary=staticmethod(text._text_retrieval_summary)
        install_tail_policy()

    @classmethod
    def tearDownClass(cls):
        cls.metrics.load_benchmark_inputs=cls.original_loader
        cls.text._text_retrieval_summary=cls.original_summary

    def fixture(self, directory, count):
        import importlib.util
        spec=importlib.util.spec_from_file_location('hfd_tail_pack',WORK/'evaluate.py')
        module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
        motion=np.zeros((51,36),np.float32);motion[:,31]=.8;motion[:,32]=1
        np.savez(directory/'source.npz',absolute=motion)
        rows=[dict(sample_id='source',score_start=0,score_end=51,caption='stand',
                   dataset_index=i,reference_file='source.npz') for i in range(count)]
        # Generated file is shared here, but packed identities must be unique.
        for i,row in enumerate(rows):
            row['sample_id']=f'sample_{i}'
            np.savez(directory/(row['sample_id']+'.npz'),absolute=motion)
        module.pack(rows,directory,directory,directory/'packed')
        return dict(task='text',generated_path=str(directory/'packed/generated.npz'),
                    reference_path=str(directory/'packed/reference.npz'),
                    condition_path=str(directory/'packed/conditions.npz'),
                    expected_frames=31,expected_fps=30)

    def test_complete_batch_unchanged_and_all_33_retained(self):
        with tempfile.TemporaryDirectory() as tmp:
            root=Path(tmp);args=self.fixture(root,32)
            old=self.original_loader(**args);new=self.metrics.load_benchmark_inputs(**args)
            self.assertEqual(old.keys(),new.keys())
            for key in old:np.testing.assert_equal(old[key],new[key])
            args=self.fixture(root,33)
            with self.assertRaisesRegex(ValueError,'batch-32'):self.original_loader(**args)
            self.assertEqual(self.metrics.load_benchmark_inputs(**args)['num_samples'],33)
            with self.assertRaisesRegex(ValueError,'batch-32'):
                self.metrics.load_benchmark_inputs(**self.fixture(root,31))

    def test_tail_still_validates_identity_caption_quaternion_and_lengths(self):
        with tempfile.TemporaryDirectory() as tmp:
            root=Path(tmp);args=self.fixture(root,33);path=Path(args['generated_path'])
            with np.load(path) as z:original={k:z[k] for k in z.files}
            for field in ('sample_id','captions','qpos_36','lengths'):
                changed={k:v.copy() for k,v in original.items()}
                if field in ('sample_id','captions'):changed[field][-1]='BAD'
                elif field=='qpos_36':changed[field][-1,0,3:7]=0
                else:changed[field][-1]-=1
                np.savez(path,**changed)
                with self.subTest(field=field),self.assertRaises(ValueError):
                    self.metrics.load_benchmark_inputs(**args)

    def test_native_retrieval_formula_and_full_matching_preserved(self):
        rng=np.random.default_rng(42)
        motion=rng.normal(size=(33,16)).astype(np.float32);text=motion.copy();text[-1]+=5
        expected=self.original_summary(motion,text,require_full_batches=False)
        actual=self.text._text_retrieval_summary(motion,text,require_full_batches=True)
        self.assertEqual(actual,expected)
        self.assertEqual(actual['retrieval_num_samples'],32)
        self.assertAlmostEqual(actual['matching_score'],float(np.linalg.norm(motion-text,axis=-1).mean()),places=6)
        self.assertEqual(self.text._text_retrieval_summary(motion[:32],text[:32]),
                         self.original_summary(motion[:32],text[:32]))


if __name__=='__main__':unittest.main()