Download benchmark/evaluate.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/evaluate.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/benchmark/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/benchmark/evaluate.py
14.1 kB
| """Pack HFD motion segments for the unchanged OmniHM Text/Speech evaluator. | |
| Also score the complete idle/listen/motion episode and each GT-defined phase. | |
| Physics is evaluated with the same FK, contact thresholds and jerk definition. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| from common import WORK, OMG, OMNI, EVALUATOR, PEER, qpos_from_absolute, sha256, write_json | |
| from session_data import flattened_rows | |
| import numpy as np | |
| def verify_scores(cohort, generated, output): | |
| """Verify the complete score bundle against the frozen cohort, not file presence.""" | |
| def read(path): | |
| return json.loads(Path(path).read_text()) | |
| def require(ok, message): | |
| if not ok:raise ValueError(message) | |
| manifest=read(cohort/'manifest.json');identity=read(generated/'identity.json') | |
| protocol=read(output/'protocol.json');native=output/'motion' | |
| rows=flattened_rows(manifest['rows']);n=len(rows) | |
| ids=np.asarray([r['sample_id'] for r in rows]) | |
| require(identity['cohort_sha256']==sha256(cohort/'manifest.json'),'Generated cohort differs') | |
| require(all(protocol.get(k)==v for k,v in identity.items()),'Metric generation identity differs') | |
| require(protocol['count']==n and protocol['session_count']==len(manifest['rows']),'Metric counts differ') | |
| state=read(native/'run_state.json') | |
| require(state.get('status')=='complete' and state.get('kind')=='benchmark' | |
| and state.get('paper_row_complete') is True,'Native scoring is incomplete') | |
| raw=read(native/'metrics.json');paper=read(native/'paper_metrics.json') | |
| require(raw['protocol']['num_samples']==n,'Native sample count differs') | |
| require(paper.get('complete') is True and len(paper['values'])==14 | |
| and np.isfinite(np.asarray(paper['values'],float)).all(),'Incomplete/nonfinite paper row') | |
| sys.path[:0]=[str(OMNI),str(OMG/'src')] | |
| from metrics import build_paper_payload | |
| require(paper==build_paper_payload('text',raw),'Displayed metrics differ from raw native results') | |
| tracker=raw['tracker_executed'] | |
| require(tracker.get('enabled') is True and tracker.get('paper_eligible') is True | |
| and tracker['num_samples']==n,'Missing/noncanonical/partial tracking') | |
| for key in ('generated_retrieval_diagnostics','reference_retrieval_diagnostics'): | |
| require(raw['generated_target']['text'][key]['retrieval_num_samples']==n//32*32, | |
| 'Retrieval tail count differs') | |
| arrays={ | |
| 'sample_metrics.npz':('contact_sliding_speed','body_jerk_mean'), | |
| 'tracker_failure_flags.npz':('mpjpe','g_mpjpe','e_vel','e_acc','fall','joint_limit'), | |
| 'embeddings.npz':('reference_embeddings','generated_embeddings','text_embeddings', | |
| 'distribution_reference_embeddings','distribution_generated_embeddings')} | |
| for name,required in arrays.items(): | |
| with np.load(native/name,allow_pickle=False) as z: | |
| require(np.array_equal(z['sample_id'],ids),f'{name}: sample identities/order differ') | |
| require(set(required)<=set(z.files),f'{name}: missing metric arrays') | |
| require(all(z[k].dtype.kind in 'biufc' for k in required),f'{name}: nonnumeric metric array') | |
| for key in z.files: | |
| value=z[key] | |
| if value.dtype.kind in 'biufc': | |
| require(value.ndim>=1 and len(value)==n and np.isfinite(value).all(), | |
| f'{name}: invalid array {key}') | |
| with np.load(output/'inputs/conditions.npz',allow_pickle=False) as z: | |
| require(np.array_equal(z['sample_id'],ids),'Packed input identities differ') | |
| require(z['captions'].tolist()==[r['caption'] for r in rows],'Final-command captions differ') | |
| phase=read(output/'phase_physics.json') | |
| require(phase['summary']['all']['sources']==len(manifest['rows']),'Incomplete full-session physics') | |
| all_rows=[r for r in phase['per_segment'] if r['phase']=='all'] | |
| require(len(all_rows)==len(manifest['rows']) and {r['sample_id'] for r in all_rows} | |
| =={r['sample_id'] for r in manifest['rows']},'Full-session physics identities differ') | |
| for row in phase['per_segment']: | |
| require(all(np.isfinite(v) for k in ('generated','gt') for v in row[k].values()), | |
| 'Nonfinite phase physics') | |
| require(set(protocol['source_hashes'])=={str(p) for p in (OMNI/'evaluate.py',OMNI/'metrics.py',EVALUATOR)}, | |
| 'Missing native metric/encoder provenance') | |
| for path,expected in protocol['source_hashes'].items(): | |
| require(sha256(path)==expected,'Metric implementation/encoder changed: '+path) | |
| adapter=read(native/'hfd_adapter.json') | |
| require(adapter['adapter_sha256']==sha256(WORK/'native_eval.py'),'Native adapter changed') | |
| receipt=dict(complete=True,session_count=len(manifest['rows']),motion_segments=n, | |
| retrieval_count=n//32*32,cohort_sha256=identity['cohort_sha256'], | |
| paper_metrics_sha256=sha256(native/'paper_metrics.json'), | |
| scope='Completion/identity/finite-value check; no extra generation or score changes') | |
| write_json(output/'result_audit.json',receipt) | |
| return receipt | |
| def pack(rows, cohort, generated, destination): | |
| sys.path.insert(0,str(OMG/'src')) | |
| from omg.robots.g1.constants import G1_JOINT_NAMES | |
| pred=[];gt=[] | |
| for row in rows: | |
| s,e=row['score_start'],row['score_end'] | |
| with np.load(generated/(row.get('parent_sample_id',row['sample_id'])+'.npz')) as z:p=z['absolute'] | |
| with np.load(cohort/row['reference_file']) as z:g=z['absolute'] | |
| if len(p)<e or len(g)<e:raise ValueError('Truncated motion segment') | |
| pred.append(qpos_from_absolute(p[s:e]));gt.append(qpos_from_absolute(g[s:e])) | |
| lengths=np.array([len(x) for x in gt],np.int32) | |
| if [len(x) for x in pred]!=lengths.tolist():raise ValueError('Mismatched timing') | |
| n=int(lengths.max()) | |
| common=dict(fps=np.float32(30),valid=np.arange(n)[None]<lengths[:,None],lengths=lengths, | |
| sample_id=np.array([r['sample_id'] for r in rows]),joint_names=np.array(G1_JOINT_NAMES), | |
| captions=np.array([r['caption'] for r in rows]),dataset=np.array(['hfd-val']*len(rows)), | |
| dataset_index=np.array([r['dataset_index'] for r in rows],np.int32)) | |
| destination.mkdir(parents=True,exist_ok=True) | |
| for name,values in [('generated',pred),('reference',gt)]: | |
| padded=np.stack([np.concatenate([x,np.repeat(x[-1:],n-len(x),axis=0)]) for x in values]) | |
| np.savez_compressed(destination/f'{name}.npz',qpos_36=padded,**common) | |
| np.savez_compressed(destination/'conditions.npz',sample_id=common['sample_id'], | |
| captions=np.array([r['caption'] for r in rows]),dataset=np.array(['hfd-val']*len(rows)), | |
| dataset_index=np.array([r['dataset_index'] for r in rows],np.int32)) | |
| return n | |
| def phase_physics(rows,cohort,generated,output): | |
| import torch | |
| sys.path.insert(0,str(OMG/'src')) | |
| from omg.generation.metrics.physical import physical_qpos_metrics | |
| from omg.robots.g1.kinematics import G1Kinematics | |
| rep=SimpleNamespace(kinematics=G1Kinematics(OMG/'assets/robots/g1/g1_kinematics.json').cuda().eval()) | |
| keys=['contact_sliding_speed','body_jerk_mean'] | |
| records=[] | |
| for row in rows: | |
| with np.load(generated/(row['sample_id']+'.npz')) as z:p=z['absolute'] | |
| with np.load(cohort/row['reference_file']) as z:g=z['absolute'];labels=z['controls'] | |
| # Tail shorter than one emission is not silently padded or time-warped. | |
| n=min(len(p),len(g)) | |
| qpred=qpos_from_absolute(p[:n]);qgt=qpos_from_absolute(g[:n]) | |
| time=np.arange(len(qpred))/30 | |
| states=labels[np.minimum(np.rint(time*50).astype(int),n-1)] | |
| for phase,label in [('all',None),('idle',2),('listen',0),('motion',1)]: | |
| mask=np.ones(len(qpred),bool) if label is None else states==label | |
| # Split discontinuities: never differentiate across distinct phase intervals. | |
| boundaries=np.flatnonzero(np.diff(np.r_[False,mask,False].astype(int))).reshape(-1,2) | |
| for start,end in boundaries: | |
| if end-start<4:continue | |
| record=dict(sample_id=row['sample_id'],phase=phase,frames=int(end-start), | |
| start_frame=int(start),end_frame=int(end),fps=30) | |
| for name,q in [('generated',qpred),('gt',qgt)]: | |
| x=torch.from_numpy(q[start:end][None]).cuda() | |
| with torch.inference_mode(): | |
| value=physical_qpos_metrics(qpos_36=x,representation=rep,fps=30., | |
| valid=torch.ones(x.shape[:2],device=x.device,dtype=torch.bool), | |
| contact_height_threshold=.12,contact_penetration_tolerance=.02) | |
| record[name]={k:float(value[k].cpu()) for k in keys} | |
| records.append(record) | |
| # First aggregate segments per source, then give each source equal weight. | |
| summary={} | |
| for phase in ['all','idle','listen','motion']: | |
| subset=[r for r in records if r['phase']==phase] | |
| ids=sorted(set(r['sample_id'] for r in subset)) | |
| summary[phase]=dict(sources=len(ids),frames=sum(r['frames'] for r in subset)) | |
| for name in ['generated','gt']: | |
| summary[phase][name]={} | |
| for key in keys: | |
| values=[] | |
| for sid in ids: | |
| same=[r for r in subset if r['sample_id']==sid] | |
| values.append(np.average([r[name][key] for r in same],weights=[r['frames'] for r in same])) | |
| summary[phase][name][key]=dict(mean=float(np.mean(values)),std=float(np.std(values))) if values else None | |
| write_json(output,dict(fps=30,units=dict(contact_sliding_speed='m/s',body_jerk_mean='m/s^3'), | |
| phase_source='fixed GT timeline; never predicted state', | |
| aggregation='frame-weighted phase segments within source, then equal source mean; ddof=0', | |
| note='All includes transitions. Short <4-frame phase segments excluded only from phase summaries.', | |
| summary=summary,per_segment=records)) | |
| def main(): | |
| p=argparse.ArgumentParser() | |
| p.add_argument('--cohort',type=Path,default=WORK/'outputs/sessions2048') | |
| p.add_argument('--generated',type=Path,default=WORK/'outputs/session_ckpt_1800000/generated') | |
| p.add_argument('--output',type=Path,default=WORK/'outputs/session_ckpt_1800000/eval') | |
| p.add_argument('--tracker',choices=['run','skip'],default='run') | |
| p.add_argument('--verify-only',action='store_true',help='Check a finished score bundle without re-running inference/evaluation') | |
| a=p.parse_args() | |
| a.cohort=a.cohort.resolve();a.generated=a.generated.resolve();a.output=a.output.resolve() | |
| if a.verify_only: | |
| print(json.dumps(verify_scores(a.cohort,a.generated,a.output),ensure_ascii=False)) | |
| return | |
| a.output.mkdir(parents=True,exist_ok=True) | |
| manifest=json.loads((a.cohort/'manifest.json').read_text()) | |
| identity=json.loads((a.generated/'identity.json').read_text()) | |
| if identity['cohort_sha256']!=sha256(a.cohort/'manifest.json'):raise ValueError('Wrong cohort') | |
| rows=manifest['rows'] | |
| for row in rows: | |
| meta=json.loads((a.generated/(row['sample_id']+'.json')).read_text()) | |
| if any(meta.get(k)!=v for k,v in identity.items()):raise ValueError('Mixed generation identity') | |
| packdir=a.output/'inputs' | |
| segments=flattened_rows(rows) | |
| frames=pack(segments,a.cohort,a.generated,packdir) | |
| write_json(a.output/'scored_segments.json',[{k:r[k] for k in ('sample_id','parent_sample_id','turn_index','caption','task','score_start','score_end') if k in r} for r in segments]) | |
| phase_physics(rows,a.cohort,a.generated,a.output/'phase_physics.json') | |
| options={ | |
| 'task':'text','generated-path':packdir/'generated.npz','reference-path':packdir/'reference.npz', | |
| 'condition-path':packdir/'conditions.npz','output-dir':a.output/'motion', | |
| 'omg-root':OMG,'expected-frames':frames,'expected-fps':30,'device':'cuda','batch-size':32, | |
| 'validate-only':0,'kinematics-path':OMG/'assets/robots/g1/g1_kinematics.json', | |
| 'contact-height-threshold':.12,'contact-penetration-tolerance':.02, | |
| 'evaluator-checkpoint':EVALUATOR,'retrieval-evaluator-checkpoint':EVALUATOR, | |
| 'motion-key':'body_pos_local','t5-model':OMG/'models/t5-3b-local', | |
| 'text-cache-dir':a.output/'text_cache','text-batch-size':16,'seed':0, | |
| 'beat-direction':'music_to_motion','beat-threshold':.5,'sigma-frames':3., | |
| 'min-motion-beat-distance-seconds':.25,'tracker-mode':a.tracker, | |
| 'tracker-artifact':a.output/'unused.npz', | |
| 'holomotion-onnx':OMG/'models/HoloMotion/HoloMotion_v1.2/holomotion_v1.2_motion_tracking_model/exported/model.onnx', | |
| 'tracker-robot-xml':OMG/'assets/holomotion/g1_29dof/scene_29dof.xml', | |
| 'tracker-target-fps':50,'tracker-providers':'TensorrtExecutionProvider,CUDAExecutionProvider', | |
| 'tracker-control-substeps':10,'tracker-action-clip':10,'tracker-steps':0} | |
| command=[sys.executable,'-B',str(WORK/'native_eval.py')] | |
| for k,v in options.items():command.extend(['--'+k,str(v)]) | |
| write_json(a.output/'protocol.json',dict(**identity,conditioning='speech, not text', | |
| reference_table='Table 3 / Table 6, same motion metrics; not music metrics', | |
| session_count=len(rows),count=len(segments),small_sample_diagnostic=len(rows)<2048,direct_paper_comparison=False, | |
| retrieval_count=len(segments)//32*32,retrieval_tail_omitted=len(segments)%32, | |
| aggregation='Equal eligible source-occurrence weight; full-session physics separately. Retrieval uses complete batch32 only.', | |
| source_hashes={str(x):sha256(x) for x in [OMNI/'evaluate.py',OMNI/'metrics.py',EVALUATOR]}, | |
| command=command,scope='motion-only tracking/distribution/retrieval; full and phase physics separately')) | |
| subprocess.run(command,cwd=OMG,check=True,env={**os.environ,'PYTHONDONTWRITEBYTECODE':'1'}) | |
| if a.tracker=='run':verify_scores(a.cohort,a.generated,a.output) | |
| if __name__=='__main__':main() | |