Download code/scripts/evaluate_libero_plus.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 7.42 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/evaluate_libero_plus.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/evaluate_libero_plus.py
-
curl -L -o evaluate_libero_plus.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/evaluate_libero_plus.py
7.42 kB
| """Pinned LIBERO-Plus evaluator; stock per-step rendering preserves image corruptions.""" | |
| import argparse,collections,fcntl,hashlib,json,pathlib,sys,time,random | |
| import numpy as np | |
| import torch | |
| from robottt_pi05.config import ExperimentConfig | |
| from robottt_pi05.openpi_backend import load_model,Original | |
| from robottt_pi05.modeling import create_adapter | |
| from robottt_pi05.checkpoint import load_weights | |
| from robottt_pi05.eval_io import atomic_json,sha | |
| from robottt_pi05.live import LiberoPolicy,LegacyLiberoPolicy | |
| def main(): | |
| p=argparse.ArgumentParser();p.add_argument('--run-config',required=True);p.add_argument('--plan',required=True);p.add_argument('--output',required=True);p.add_argument('--max-episodes',type=int);p.add_argument('--render-every-step',action='store_true') | |
| a=p.parse_args();r=json.loads(pathlib.Path(a.run_config).read_text());base=pathlib.Path(a.run_config).resolve().parent | |
| def path(n):return (base/r['paths'][n]).resolve() | |
| source=path('openpi_source');sys.path[:0]=[str(source/'src'),str(source/'packages/openpi-client/src'),str(path('libero_plus_source'))] | |
| from libero.libero import benchmark,get_libero_path | |
| from libero.libero.envs import OffScreenRenderEnv | |
| from robosuite.utils.transform_utils import quat2axisangle | |
| plan=json.loads(pathlib.Path(a.plan).read_text());out=pathlib.Path(a.output);out.mkdir(parents=True,exist_ok=True) | |
| evaluation_lock=(out/'evaluation.lock').open('a') | |
| fcntl.flock(evaluation_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| proof_path=pathlib.Path(__file__).resolve().parents[1]/'.release_local/audit/query-render-parity.json' | |
| proof=json.loads(proof_path.read_text()) if proof_path.exists() else {} | |
| helper_sha=hashlib.sha256(pathlib.Path(__file__).with_name('libero_render.py').read_bytes()).hexdigest() | |
| query_render=False # Original LIBERO rendering proof does not cover Plus perturbations. | |
| from libero_render import QueryRenderGate | |
| if plan['action_execution_horizon']!=10 or plan['solver_steps']!=10:raise ValueError('Unsupported explicit execution/solver horizon') | |
| torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=True | |
| model=load_model(source=source,checkpoint=path('model_checkpoint'),profile=r['model_profile']) | |
| config=ExperimentConfig.load(path('experiment_config')) | |
| if r.get('original',False): | |
| adapter=Original(model);checkpoint_sha=sha(path('model_checkpoint')) | |
| else: | |
| adapter,slow=create_adapter(model,config,inference=True) | |
| adapter.minimal_diagnostics=True | |
| checkpoint=path('extension_checkpoint') if 'extension_checkpoint' in r['paths'] else path('output')/'checkpoints/latest.pt' | |
| checkpoint_sha=sha(checkpoint) | |
| load_weights(checkpoint,slow,config,expected_sha256=checkpoint_sha) | |
| policy_type=LegacyLiberoPolicy if r['model_profile']=='legacy_pi05_libero' else LiberoPolicy | |
| policy=policy_type(model,adapter,json.loads(path('normalization').read_text()),path('tokenizer'),seed=plan['seed']) | |
| suite=benchmark.get_benchmark_dict()[plan['suite']]();results=[] | |
| identity=dict(checkpoint_sha256=checkpoint_sha,model_sha256=sha(path('model_checkpoint')),plan_sha256=sha(a.plan),run_config_sha256=sha(a.run_config),render_mode='verified_query_sensor_gate' if query_render else 'stock_every_step') | |
| if r['model_profile']=='legacy_pi05_libero':identity['live_policy_sha256']=sha(pathlib.Path(__file__).resolve().parents[1]/'src/robottt_pi05/live.py') | |
| identity['benchmark_revision']=plan['benchmark_revision'] | |
| identity['evaluator_sha256']=sha(__file__) | |
| for case in plan['episodes'][:a.max_episodes]: | |
| key=f'task_{case["task_id"]:02d}_init_{case["init_state_id"]:03d}';record_path=out/(key+'.json') | |
| if record_path.exists(): | |
| record=json.loads(record_path.read_text()) | |
| if record['identity']!=identity:raise ValueError('Completed episode identity differs') | |
| results.append(record);continue | |
| task=suite.get_task(case['task_id']);states_path=pathlib.Path(get_libero_path('init_states'))/task.problem_folder/task.init_states_file | |
| # Trusted init-state asset in the pinned OpenPI LIBERO submodule. | |
| # Official Plus remaps virtual view/light/layout names to physical init files. | |
| from unittest.mock import patch | |
| original_load=torch.load | |
| def load_trusted_init(filename,*args,**kwargs): | |
| root=pathlib.Path(get_libero_path('init_states')).resolve() | |
| if not pathlib.Path(filename).resolve().is_relative_to(root):raise ValueError('Init file outside pinned benchmark') | |
| return original_load(filename,map_location='cpu',weights_only=False) | |
| with patch.object(benchmark.torch,'load',side_effect=load_trusted_init): | |
| states=suite.get_task_init_states(case['task_id']) | |
| states_digest=hashlib.sha256(np.asarray(states).tobytes()).hexdigest() | |
| env=OffScreenRenderEnv(bddl_file_name=str(pathlib.Path(get_libero_path('bddl_files'))/task.problem_folder/task.bddl_file),camera_heights=256,camera_widths=256,control_freq=20) | |
| begin=time.monotonic();record=dict(identity=identity,case=case,success=False,failure=None,steps=0,init_states_array_sha256=states_digest) | |
| try: | |
| np.random.seed(case['seed']);random.seed(case['seed']);env.seed(case['seed']);env.reset();obs=env.set_init_state(states[case['init_state_id']]);policy.reset(episode_seed=case['seed']) | |
| for _ in range(plan['settle_steps']):obs,_,_,_=env.step([0.]*6+[-1.]) | |
| render_gate=QueryRenderGate(env) if query_render else None | |
| queue=collections.deque() | |
| for step in range(plan['max_steps']): | |
| if not queue: | |
| images=[obs[n][::-1,::-1].copy() for n in ('agentview_image','robot0_eye_in_hand_image')] | |
| state=np.concatenate([obs['robot0_eef_pos'],quat2axisangle(obs['robot0_eef_quat']),obs['robot0_gripper_qpos']]) | |
| actions,_=policy.infer(images,state,task.language) | |
| if actions.shape!=(10,7) or not np.isfinite(actions).all():raise ValueError('Invalid live action chunk') | |
| queue.extend(actions) | |
| if render_gate is not None:render_gate.render=len(queue)==1 | |
| obs,_,done,_=env.step(queue.popleft().tolist());record['steps']=step+1 | |
| if done or env.check_success():record['success']=True;break | |
| if not record['success']:record['failure']='timeout' | |
| except Exception as error:record['failure']=type(error).__name__+': '+str(error) | |
| finally:env.close() | |
| record.update(seconds=time.monotonic()-begin,chunks=policy.chunks);atomic_json(record_path,record);results.append(record);print(json.dumps(record),flush=True) | |
| if record['failure'] and record['failure']!='timeout':raise RuntimeError('Live evaluation failed: '+record['failure']) | |
| total=len(results);successes=sum(x['success'] for x in results) | |
| by_task={} | |
| for record in results: | |
| task=by_task.setdefault(str(record['case']['task_id']),dict(attempts=0,successes=0)) | |
| task['attempts']+=1;task['successes']+=int(record['success']) | |
| atomic_json(out/'summary.json',dict(identity=identity,attempts=total,successes=successes,success_rate=successes/total if total else None,complete=total==len(plan['episodes']),by_task=by_task,episodes=results),replace=True) | |
| if __name__=='__main__':main() | |