robottt-pi05-base-comparison / code /scripts /evaluate_libero_plus.py
jisue's picture
LIBERO-Plus matched evaluation progress
e947806 verified
Raw History Blame Contribute Delete
7.42 kB
"""Pinned LIBERO-Plus evaluator; stock per-step rendering preserves image corruptions."""
import argparse,collections,fcntl,hashlib,json,pathlib,sys,time,random
import numpy as np
import torch
from robottt_pi05.config import ExperimentConfig
from robottt_pi05.openpi_backend import load_model,Original
from robottt_pi05.modeling import create_adapter
from robottt_pi05.checkpoint import load_weights
from robottt_pi05.eval_io import atomic_json,sha
from robottt_pi05.live import LiberoPolicy,LegacyLiberoPolicy
def main():
p=argparse.ArgumentParser();p.add_argument('--run-config',required=True);p.add_argument('--plan',required=True);p.add_argument('--output',required=True);p.add_argument('--max-episodes',type=int);p.add_argument('--render-every-step',action='store_true')
a=p.parse_args();r=json.loads(pathlib.Path(a.run_config).read_text());base=pathlib.Path(a.run_config).resolve().parent
def path(n):return (base/r['paths'][n]).resolve()
source=path('openpi_source');sys.path[:0]=[str(source/'src'),str(source/'packages/openpi-client/src'),str(path('libero_plus_source'))]
from libero.libero import benchmark,get_libero_path
from libero.libero.envs import OffScreenRenderEnv
from robosuite.utils.transform_utils import quat2axisangle
plan=json.loads(pathlib.Path(a.plan).read_text());out=pathlib.Path(a.output);out.mkdir(parents=True,exist_ok=True)
evaluation_lock=(out/'evaluation.lock').open('a')
fcntl.flock(evaluation_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
proof_path=pathlib.Path(__file__).resolve().parents[1]/'.release_local/audit/query-render-parity.json'
proof=json.loads(proof_path.read_text()) if proof_path.exists() else {}
helper_sha=hashlib.sha256(pathlib.Path(__file__).with_name('libero_render.py').read_bytes()).hexdigest()
query_render=False # Original LIBERO rendering proof does not cover Plus perturbations.
from libero_render import QueryRenderGate
if plan['action_execution_horizon']!=10 or plan['solver_steps']!=10:raise ValueError('Unsupported explicit execution/solver horizon')
torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=True
model=load_model(source=source,checkpoint=path('model_checkpoint'),profile=r['model_profile'])
config=ExperimentConfig.load(path('experiment_config'))
if r.get('original',False):
adapter=Original(model);checkpoint_sha=sha(path('model_checkpoint'))
else:
adapter,slow=create_adapter(model,config,inference=True)
adapter.minimal_diagnostics=True
checkpoint=path('extension_checkpoint') if 'extension_checkpoint' in r['paths'] else path('output')/'checkpoints/latest.pt'
checkpoint_sha=sha(checkpoint)
load_weights(checkpoint,slow,config,expected_sha256=checkpoint_sha)
policy_type=LegacyLiberoPolicy if r['model_profile']=='legacy_pi05_libero' else LiberoPolicy
policy=policy_type(model,adapter,json.loads(path('normalization').read_text()),path('tokenizer'),seed=plan['seed'])
suite=benchmark.get_benchmark_dict()[plan['suite']]();results=[]
identity=dict(checkpoint_sha256=checkpoint_sha,model_sha256=sha(path('model_checkpoint')),plan_sha256=sha(a.plan),run_config_sha256=sha(a.run_config),render_mode='verified_query_sensor_gate' if query_render else 'stock_every_step')
if r['model_profile']=='legacy_pi05_libero':identity['live_policy_sha256']=sha(pathlib.Path(__file__).resolve().parents[1]/'src/robottt_pi05/live.py')
identity['benchmark_revision']=plan['benchmark_revision']
identity['evaluator_sha256']=sha(__file__)
for case in plan['episodes'][:a.max_episodes]:
key=f'task_{case["task_id"]:02d}_init_{case["init_state_id"]:03d}';record_path=out/(key+'.json')
if record_path.exists():
record=json.loads(record_path.read_text())
if record['identity']!=identity:raise ValueError('Completed episode identity differs')
results.append(record);continue
task=suite.get_task(case['task_id']);states_path=pathlib.Path(get_libero_path('init_states'))/task.problem_folder/task.init_states_file
# Trusted init-state asset in the pinned OpenPI LIBERO submodule.
# Official Plus remaps virtual view/light/layout names to physical init files.
from unittest.mock import patch
original_load=torch.load
def load_trusted_init(filename,*args,**kwargs):
root=pathlib.Path(get_libero_path('init_states')).resolve()
if not pathlib.Path(filename).resolve().is_relative_to(root):raise ValueError('Init file outside pinned benchmark')
return original_load(filename,map_location='cpu',weights_only=False)
with patch.object(benchmark.torch,'load',side_effect=load_trusted_init):
states=suite.get_task_init_states(case['task_id'])
states_digest=hashlib.sha256(np.asarray(states).tobytes()).hexdigest()
env=OffScreenRenderEnv(bddl_file_name=str(pathlib.Path(get_libero_path('bddl_files'))/task.problem_folder/task.bddl_file),camera_heights=256,camera_widths=256,control_freq=20)
begin=time.monotonic();record=dict(identity=identity,case=case,success=False,failure=None,steps=0,init_states_array_sha256=states_digest)
try:
np.random.seed(case['seed']);random.seed(case['seed']);env.seed(case['seed']);env.reset();obs=env.set_init_state(states[case['init_state_id']]);policy.reset(episode_seed=case['seed'])
for _ in range(plan['settle_steps']):obs,_,_,_=env.step([0.]*6+[-1.])
render_gate=QueryRenderGate(env) if query_render else None
queue=collections.deque()
for step in range(plan['max_steps']):
if not queue:
images=[obs[n][::-1,::-1].copy() for n in ('agentview_image','robot0_eye_in_hand_image')]
state=np.concatenate([obs['robot0_eef_pos'],quat2axisangle(obs['robot0_eef_quat']),obs['robot0_gripper_qpos']])
actions,_=policy.infer(images,state,task.language)
if actions.shape!=(10,7) or not np.isfinite(actions).all():raise ValueError('Invalid live action chunk')
queue.extend(actions)
if render_gate is not None:render_gate.render=len(queue)==1
obs,_,done,_=env.step(queue.popleft().tolist());record['steps']=step+1
if done or env.check_success():record['success']=True;break
if not record['success']:record['failure']='timeout'
except Exception as error:record['failure']=type(error).__name__+': '+str(error)
finally:env.close()
record.update(seconds=time.monotonic()-begin,chunks=policy.chunks);atomic_json(record_path,record);results.append(record);print(json.dumps(record),flush=True)
if record['failure'] and record['failure']!='timeout':raise RuntimeError('Live evaluation failed: '+record['failure'])
total=len(results);successes=sum(x['success'] for x in results)
by_task={}
for record in results:
task=by_task.setdefault(str(record['case']['task_id']),dict(attempts=0,successes=0))
task['attempts']+=1;task['successes']+=int(record['success'])
atomic_json(out/'summary.json',dict(identity=identity,attempts=total,successes=successes,success_rate=successes/total if total else None,complete=total==len(plan['episodes']),by_task=by_task,episodes=results),replace=True)
if __name__=='__main__':main()