Download benchmark/test_native_eval.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/test_native_eval.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/test_native_eval.py
-
curl -L -o test_native_eval.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/test_native_eval.py
10.9 kB
| """Non-multiple-of-32 sessions must not disable native input integrity checks.""" | |
| import tempfile | |
| import unittest | |
| from unittest.mock import patch | |
| import json | |
| from pathlib import Path | |
| import numpy as np | |
| from common import WORK, OMG, OMNI, EVALUATOR, sha256, write_json | |
| from native_eval import install_tail_policy, install_bounded_motion_conversion | |
| class EntryPointTests(unittest.TestCase): | |
| def test_provenance_does_not_prepopulate_native_output(self): | |
| import native_eval | |
| with tempfile.TemporaryDirectory() as tmp: | |
| output=Path(tmp)/'new_output' | |
| def native_main(*args, **kwargs): | |
| self.assertFalse(output.exists()) | |
| output.mkdir() | |
| raise SystemExit(0) | |
| with patch.object(native_eval.sys,'argv',['native_eval.py','--output-dir',str(output)]), \ | |
| patch.object(native_eval,'install_tail_policy'), \ | |
| patch.object(native_eval,'install_bounded_motion_conversion'), \ | |
| patch.object(native_eval.runpy,'run_path',side_effect=native_main): | |
| native_eval.main() | |
| self.assertTrue((output/'hfd_adapter.json').is_file()) | |
| class BoundedFKTests(unittest.TestCase): | |
| def test_native_fk_matches_for_all_sources_frames_and_motion_keys(self): | |
| import torch | |
| from omg.benchmarks.runners import common | |
| from omg.robots.g1.kinematics import G1Kinematics | |
| kin=G1Kinematics(str(OMG/'assets/robots/g1/g1_kinematics.json')).eval() | |
| rng=torch.Generator().manual_seed(42) | |
| qpos=torch.randn((33,9,36),generator=rng)*.1 | |
| qpos[...,2]+=.8 | |
| quat=torch.randn((33,9,4),generator=rng) | |
| qpos[...,3:7]=quat/torch.linalg.vector_norm(quat,dim=-1,keepdim=True) | |
| original=common._motion_for_evaluator | |
| try: | |
| install_bounded_motion_conversion() | |
| with torch.inference_mode(): | |
| for key in ('qpos_36','body_pos_local','body_link_pos_local'): | |
| expected=original(qpos,key,kinematics=kin) | |
| actual=common._motion_for_evaluator(qpos,key,kinematics=kin) | |
| torch.testing.assert_close(actual,expected,rtol=0,atol=0) | |
| finally: | |
| common._motion_for_evaluator=original | |
| class ScoreBundleTests(unittest.TestCase): | |
| def fixture(self, root): | |
| import importlib.util | |
| from metrics import build_paper_payload | |
| from session_data import flattened_rows | |
| spec=importlib.util.spec_from_file_location('hfd_score_check',WORK/'evaluate.py') | |
| module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) | |
| cohort,generated,output=(root/x for x in ('cohort','generated','eval')) | |
| native=output/'motion';native.mkdir(parents=True) | |
| original=[dict(sample_id=f's{i}',caption='stand',reference_frames=51) for i in range(32)] | |
| write_json(cohort/'manifest.json',dict(rows=original)) | |
| rows=flattened_rows(original);ids=np.array([r['sample_id'] for r in rows]) | |
| identity=dict(step=1,cohort_sha256=sha256(cohort/'manifest.json')) | |
| write_json(generated/'identity.json',identity) | |
| write_json(output/'protocol.json',dict(**identity,count=32,session_count=32, | |
| source_hashes={str(p):sha256(p) for p in (OMNI/'evaluate.py',OMNI/'metrics.py',EVALUATOR)})) | |
| write_json(native/'run_state.json',dict(status='complete',kind='benchmark',paper_row_complete=True)) | |
| raw=dict(protocol=dict(num_samples=32),generated_target=dict( | |
| text=dict(motion_fid=.1,matching_score=1.,r_precision=[.3,.5,.7],diversity_generated=1.2, | |
| generated_retrieval_diagnostics=dict(retrieval_num_samples=32), | |
| reference_retrieval_diagnostics=dict(retrieval_num_samples=32)), | |
| physical=dict(contact_sliding_speed=dict(mean=.1),body_jerk_mean=dict(mean=20.))), | |
| tracker_executed=dict(enabled=True,paper_eligible=True,num_samples=32, | |
| metrics={k:dict(mean=1.) for k in ('mpjpe','g_mpjpe','e_vel','e_acc')}, | |
| failures=dict(fall_rate_percent=0.,joint_limit_rate_percent=0.))) | |
| write_json(native/'metrics.json',raw) | |
| write_json(native/'paper_metrics.json',build_paper_payload('text',raw)) | |
| np.savez(native/'sample_metrics.npz',sample_id=ids, | |
| contact_sliding_speed=np.full(32,.1),body_jerk_mean=np.full(32,20.)) | |
| np.savez(native/'tracker_failure_flags.npz',sample_id=ids, | |
| **{k:np.ones(32) for k in ('mpjpe','g_mpjpe','e_vel','e_acc')}, | |
| fall=np.zeros(32,bool),joint_limit=np.zeros(32,bool)) | |
| np.savez(native/'embeddings.npz',sample_id=ids, | |
| **{k:np.ones((32,512),np.float32) for k in ('reference_embeddings','generated_embeddings', | |
| 'text_embeddings','distribution_reference_embeddings','distribution_generated_embeddings')}) | |
| (output/'inputs').mkdir() | |
| np.savez(output/'inputs/conditions.npz',sample_id=ids,captions=np.array(['stand']*32)) | |
| write_json(output/'phase_physics.json',dict(summary=dict(all=dict(sources=32)), | |
| per_segment=[dict(sample_id=r['sample_id'],phase='all',frames=31, | |
| generated=dict(jerk=1.),gt=dict(jerk=1.)) for r in rows])) | |
| write_json(native/'hfd_adapter.json',dict(adapter_sha256=sha256(WORK/'native_eval.py'))) | |
| return module.verify_scores,(cohort,generated,output) | |
| def test_complete_bundle_and_incomplete_status(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| verify,args=self.fixture(Path(tmp));out=args[-1] | |
| self.assertTrue(verify(*args)['complete']) | |
| state=out/'motion/run_state.json' | |
| write_json(state,dict(status='running',kind='benchmark',paper_row_complete=True)) | |
| with self.assertRaisesRegex(ValueError,'incomplete'):verify(*args) | |
| def test_mixed_order_nonfinite_and_inconsistent_table_rejected(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| verify,args=self.fixture(Path(tmp));out=args[-1];native=out/'motion' | |
| path=native/'embeddings.npz' | |
| with np.load(path) as z:original={k:z[k] for k in z.files} | |
| np.savez(path,**{**original,'sample_id':original['sample_id'][::-1]}) | |
| with self.assertRaisesRegex(ValueError,'identities/order'):verify(*args) | |
| values=original['generated_embeddings'].copy();values[-1]=np.nan | |
| np.savez(path,**{**original,'generated_embeddings':values}) | |
| with self.assertRaisesRegex(ValueError,'invalid array'):verify(*args) | |
| np.savez(path,**original) | |
| paper=json.loads((native/'paper_metrics.json').read_text());paper['values'][0]+=1 | |
| write_json(native/'paper_metrics.json',paper) | |
| with self.assertRaisesRegex(ValueError,'Displayed metrics'):verify(*args) | |
| class TailPolicyTests(unittest.TestCase): | |
| def setUpClass(cls): | |
| import metrics | |
| from omg.benchmarks.runners import text | |
| cls.metrics,cls.text=metrics,text | |
| cls.original_loader=staticmethod(metrics.load_benchmark_inputs) | |
| cls.original_summary=staticmethod(text._text_retrieval_summary) | |
| install_tail_policy() | |
| def tearDownClass(cls): | |
| cls.metrics.load_benchmark_inputs=cls.original_loader | |
| cls.text._text_retrieval_summary=cls.original_summary | |
| def fixture(self, directory, count): | |
| import importlib.util | |
| spec=importlib.util.spec_from_file_location('hfd_tail_pack',WORK/'evaluate.py') | |
| module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module) | |
| motion=np.zeros((51,36),np.float32);motion[:,31]=.8;motion[:,32]=1 | |
| np.savez(directory/'source.npz',absolute=motion) | |
| rows=[dict(sample_id='source',score_start=0,score_end=51,caption='stand', | |
| dataset_index=i,reference_file='source.npz') for i in range(count)] | |
| # Generated file is shared here, but packed identities must be unique. | |
| for i,row in enumerate(rows): | |
| row['sample_id']=f'sample_{i}' | |
| np.savez(directory/(row['sample_id']+'.npz'),absolute=motion) | |
| module.pack(rows,directory,directory,directory/'packed') | |
| return dict(task='text',generated_path=str(directory/'packed/generated.npz'), | |
| reference_path=str(directory/'packed/reference.npz'), | |
| condition_path=str(directory/'packed/conditions.npz'), | |
| expected_frames=31,expected_fps=30) | |
| def test_complete_batch_unchanged_and_all_33_retained(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| root=Path(tmp);args=self.fixture(root,32) | |
| old=self.original_loader(**args);new=self.metrics.load_benchmark_inputs(**args) | |
| self.assertEqual(old.keys(),new.keys()) | |
| for key in old:np.testing.assert_equal(old[key],new[key]) | |
| args=self.fixture(root,33) | |
| with self.assertRaisesRegex(ValueError,'batch-32'):self.original_loader(**args) | |
| self.assertEqual(self.metrics.load_benchmark_inputs(**args)['num_samples'],33) | |
| with self.assertRaisesRegex(ValueError,'batch-32'): | |
| self.metrics.load_benchmark_inputs(**self.fixture(root,31)) | |
| def test_tail_still_validates_identity_caption_quaternion_and_lengths(self): | |
| with tempfile.TemporaryDirectory() as tmp: | |
| root=Path(tmp);args=self.fixture(root,33);path=Path(args['generated_path']) | |
| with np.load(path) as z:original={k:z[k] for k in z.files} | |
| for field in ('sample_id','captions','qpos_36','lengths'): | |
| changed={k:v.copy() for k,v in original.items()} | |
| if field in ('sample_id','captions'):changed[field][-1]='BAD' | |
| elif field=='qpos_36':changed[field][-1,0,3:7]=0 | |
| else:changed[field][-1]-=1 | |
| np.savez(path,**changed) | |
| with self.subTest(field=field),self.assertRaises(ValueError): | |
| self.metrics.load_benchmark_inputs(**args) | |
| def test_native_retrieval_formula_and_full_matching_preserved(self): | |
| rng=np.random.default_rng(42) | |
| motion=rng.normal(size=(33,16)).astype(np.float32);text=motion.copy();text[-1]+=5 | |
| expected=self.original_summary(motion,text,require_full_batches=False) | |
| actual=self.text._text_retrieval_summary(motion,text,require_full_batches=True) | |
| self.assertEqual(actual,expected) | |
| self.assertEqual(actual['retrieval_num_samples'],32) | |
| self.assertAlmostEqual(actual['matching_score'],float(np.linalg.norm(motion-text,axis=-1).mean()),places=6) | |
| self.assertEqual(self.text._text_retrieval_summary(motion[:32],text[:32]), | |
| self.original_summary(motion[:32],text[:32])) | |
| if __name__=='__main__':unittest.main() | |