File size: 7,424 Bytes
240faaa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e947806
 
 
 
 
 
 
 
240faaa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
"""Pinned LIBERO-Plus evaluator; stock per-step rendering preserves image corruptions."""
import argparse,collections,fcntl,hashlib,json,pathlib,sys,time,random
import numpy as np
import torch
from robottt_pi05.config import ExperimentConfig
from robottt_pi05.openpi_backend import load_model,Original
from robottt_pi05.modeling import create_adapter
from robottt_pi05.checkpoint import load_weights
from robottt_pi05.eval_io import atomic_json,sha
from robottt_pi05.live import LiberoPolicy,LegacyLiberoPolicy

def main():
    p=argparse.ArgumentParser();p.add_argument('--run-config',required=True);p.add_argument('--plan',required=True);p.add_argument('--output',required=True);p.add_argument('--max-episodes',type=int);p.add_argument('--render-every-step',action='store_true')
    a=p.parse_args();r=json.loads(pathlib.Path(a.run_config).read_text());base=pathlib.Path(a.run_config).resolve().parent
    def path(n):return (base/r['paths'][n]).resolve()
    source=path('openpi_source');sys.path[:0]=[str(source/'src'),str(source/'packages/openpi-client/src'),str(path('libero_plus_source'))]
    from libero.libero import benchmark,get_libero_path
    from libero.libero.envs import OffScreenRenderEnv
    from robosuite.utils.transform_utils import quat2axisangle
    plan=json.loads(pathlib.Path(a.plan).read_text());out=pathlib.Path(a.output);out.mkdir(parents=True,exist_ok=True)
    evaluation_lock=(out/'evaluation.lock').open('a')
    fcntl.flock(evaluation_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    proof_path=pathlib.Path(__file__).resolve().parents[1]/'.release_local/audit/query-render-parity.json'
    proof=json.loads(proof_path.read_text()) if proof_path.exists() else {}
    helper_sha=hashlib.sha256(pathlib.Path(__file__).with_name('libero_render.py').read_bytes()).hexdigest()
    query_render=False  # Original LIBERO rendering proof does not cover Plus perturbations.
    from libero_render import QueryRenderGate
    if plan['action_execution_horizon']!=10 or plan['solver_steps']!=10:raise ValueError('Unsupported explicit execution/solver horizon')
    torch.set_num_threads(4);torch.backends.cuda.matmul.allow_tf32=True
    model=load_model(source=source,checkpoint=path('model_checkpoint'),profile=r['model_profile'])
    config=ExperimentConfig.load(path('experiment_config'))
    if r.get('original',False):
        adapter=Original(model);checkpoint_sha=sha(path('model_checkpoint'))
    else:
        adapter,slow=create_adapter(model,config,inference=True)
        adapter.minimal_diagnostics=True
        checkpoint=path('extension_checkpoint') if 'extension_checkpoint' in r['paths'] else path('output')/'checkpoints/latest.pt'
        checkpoint_sha=sha(checkpoint)
        load_weights(checkpoint,slow,config,expected_sha256=checkpoint_sha)
    policy_type=LegacyLiberoPolicy if r['model_profile']=='legacy_pi05_libero' else LiberoPolicy
    policy=policy_type(model,adapter,json.loads(path('normalization').read_text()),path('tokenizer'),seed=plan['seed'])
    suite=benchmark.get_benchmark_dict()[plan['suite']]();results=[]
    identity=dict(checkpoint_sha256=checkpoint_sha,model_sha256=sha(path('model_checkpoint')),plan_sha256=sha(a.plan),run_config_sha256=sha(a.run_config),render_mode='verified_query_sensor_gate' if query_render else 'stock_every_step')
    if r['model_profile']=='legacy_pi05_libero':identity['live_policy_sha256']=sha(pathlib.Path(__file__).resolve().parents[1]/'src/robottt_pi05/live.py')
    identity['benchmark_revision']=plan['benchmark_revision']
    identity['evaluator_sha256']=sha(__file__)
    for case in plan['episodes'][:a.max_episodes]:
        key=f'task_{case["task_id"]:02d}_init_{case["init_state_id"]:03d}';record_path=out/(key+'.json')
        if record_path.exists():
            record=json.loads(record_path.read_text())
            if record['identity']!=identity:raise ValueError('Completed episode identity differs')
            results.append(record);continue
        task=suite.get_task(case['task_id']);states_path=pathlib.Path(get_libero_path('init_states'))/task.problem_folder/task.init_states_file
        # Trusted init-state asset in the pinned OpenPI LIBERO submodule.
        # Official Plus remaps virtual view/light/layout names to physical init files.
        from unittest.mock import patch
        original_load=torch.load
        def load_trusted_init(filename,*args,**kwargs):
            root=pathlib.Path(get_libero_path('init_states')).resolve()
            if not pathlib.Path(filename).resolve().is_relative_to(root):raise ValueError('Init file outside pinned benchmark')
            return original_load(filename,map_location='cpu',weights_only=False)
        with patch.object(benchmark.torch,'load',side_effect=load_trusted_init):
            states=suite.get_task_init_states(case['task_id'])
        states_digest=hashlib.sha256(np.asarray(states).tobytes()).hexdigest()
        env=OffScreenRenderEnv(bddl_file_name=str(pathlib.Path(get_libero_path('bddl_files'))/task.problem_folder/task.bddl_file),camera_heights=256,camera_widths=256,control_freq=20)
        begin=time.monotonic();record=dict(identity=identity,case=case,success=False,failure=None,steps=0,init_states_array_sha256=states_digest)
        try:
            np.random.seed(case['seed']);random.seed(case['seed']);env.seed(case['seed']);env.reset();obs=env.set_init_state(states[case['init_state_id']]);policy.reset(episode_seed=case['seed'])
            for _ in range(plan['settle_steps']):obs,_,_,_=env.step([0.]*6+[-1.])
            render_gate=QueryRenderGate(env) if query_render else None
            queue=collections.deque()
            for step in range(plan['max_steps']):
                if not queue:
                    images=[obs[n][::-1,::-1].copy() for n in ('agentview_image','robot0_eye_in_hand_image')]
                    state=np.concatenate([obs['robot0_eef_pos'],quat2axisangle(obs['robot0_eef_quat']),obs['robot0_gripper_qpos']])
                    actions,_=policy.infer(images,state,task.language)
                    if actions.shape!=(10,7) or not np.isfinite(actions).all():raise ValueError('Invalid live action chunk')
                    queue.extend(actions)
                if render_gate is not None:render_gate.render=len(queue)==1
                obs,_,done,_=env.step(queue.popleft().tolist());record['steps']=step+1
                if done or env.check_success():record['success']=True;break
            if not record['success']:record['failure']='timeout'
        except Exception as error:record['failure']=type(error).__name__+': '+str(error)
        finally:env.close()
        record.update(seconds=time.monotonic()-begin,chunks=policy.chunks);atomic_json(record_path,record);results.append(record);print(json.dumps(record),flush=True)
        if record['failure'] and record['failure']!='timeout':raise RuntimeError('Live evaluation failed: '+record['failure'])
    total=len(results);successes=sum(x['success'] for x in results)
    by_task={}
    for record in results:
        task=by_task.setdefault(str(record['case']['task_id']),dict(attempts=0,successes=0))
        task['attempts']+=1;task['successes']+=int(record['success'])
    atomic_json(out/'summary.json',dict(identity=identity,attempts=total,successes=successes,success_rate=successes/total if total else None,complete=total==len(plan['episodes']),by_task=by_task,episodes=results),replace=True)

if __name__=='__main__':main()