robottt-pi05-base-comparison / code /scripts /run_libero_plus.py
jisue's picture
LIBERO-Plus matched evaluation progress
112e766 verified
Raw History Blame Contribute Delete
10.5 kB
"""Detached matched FT/L4/L18 LIBERO-Plus-10 evaluation, resumable by episode."""
import collections
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import zipfile
import launch_comparison as pipeline
ROOT, LOCAL = pipeline.ROOT, pipeline.LOCAL
SOURCE = Path('/tmp/robottt-libero-plus')
ARCHIVE = Path('/tmp/robottt-libero-plus-assets.zip')
REVISION = '4976dc30028e805ff8094b55501d532c48fec182'
ENV = {**pipeline.ENV, 'LIBERO_CONFIG_PATH': str(LOCAL/'libero-plus-config'),
'PYTHONPATH': str(SOURCE)+':'+pipeline.ENV['PYTHONPATH']}
LABELS = ('Original','L4','L18')
def read(path):
return json.loads(path.read_text()) if path.exists() else {}
def status(stage, **kw):
pipeline.write(LOCAL/'libero-plus-status.json', dict(stage=stage,pid=os.getpid(),updated_unix_time=time.time(),**kw))
def notify(message):
from run_legacy_libero import notify as send
send(message)
def backup(final=False):
# Small incremental commits; the existing worker separately mirrors weights and code.
from huggingface_hub import HfApi, CommitOperationAdd
c=read(LOCAL/'hf-backup.json'); token=Path(c['token_file']).read_text().strip(); api=HfApi(token=token)
files={}
for path in LOCAL.glob('libero-plus-*.json'): files['progress/'+path.name]=path.read_bytes()
for path in (ROOT/'scripts').glob('*libero_plus.py'): files['code/scripts/'+path.name]=path.read_bytes()
for path in (LOCAL/'evaluations').glob('libero-plus-*/*.json'): files[str(path.relative_to(LOCAL))]=path.read_bytes()
commit=api.create_commit(repo_id=c['repo_id'],operations=[CommitOperationAdd(path_in_repo=k,path_or_fileobj=v) for k,v in files.items()],commit_message='LIBERO-Plus matched evaluation '+('final' if final else 'progress'))
if final:
from huggingface_hub import hf_hub_download
for name,expected in files.items():
actual=Path(hf_hub_download(c['repo_id'],name,revision=commit.oid,token=token,cache_dir='/tmp/robottt-plus-verify')).read_bytes()
assert actual==expected,name
pipeline.write(LOCAL/'libero-plus-backup.json',dict(commit_oid=commit.oid,files=len(files),remote_readback_verified=final,updated_unix_time=time.time()))
def start(label, maximum=None):
command=[str(pipeline.PYTHON),'scripts/evaluate_libero_plus.py','--run-config',str(LOCAL/f'libero-plus-{label}.json'),'--plan',str(LOCAL/'libero-plus-plan.json'),'--output',str(LOCAL/'evaluations'/f'libero-plus-{label}')]
if maximum is not None:command+=['--max-episodes',str(maximum)]
with (LOCAL/'audit'/f'libero-plus-{label}.log').open('a') as log:
return subprocess.Popen(command,cwd=ROOT,env=ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT)
def main():
lock=(LOCAL/'libero-plus-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
revision=subprocess.check_output(['git','-C',str(SOURCE),'rev-parse','HEAD'],text=True).strip()
assert revision==REVISION
if not (LOCAL/'libero-plus-assets.json').exists() or not (SOURCE/'libero/libero/assets').is_dir():
# Wait for the separately launched download to finish, including central directory.
last_backup=0
while not zipfile.is_zipfile(ARCHIVE):
# Adopt an existing download; only resume if no process still writes this file.
writers=[]
for proc in Path('/proc').glob('[0-9]*/cmdline'):
try:
args=proc.read_bytes().split(b'\0')
if b'curl' in args and str(ARCHIVE).encode() in args:writers.append(int(proc.parent.name))
except (FileNotFoundError,PermissionError):pass
if not writers:
with Path('/tmp/robottt-plus-download.log').open('a') as log:
child=subprocess.Popen(['curl','-fL','--retry','5','-C','-','https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip','-o',str(ARCHIVE)],stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
writers=[child.pid]
status('downloading-assets',downloaded_bytes=ARCHIVE.stat().st_size if ARCHIVE.exists() else 0,download_pids=writers)
if time.time()-last_backup>300:
try:backup();last_backup=time.time()
except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
time.sleep(30)
with zipfile.ZipFile(ARCHIVE) as archive:
unpacked=sum(x.file_size for x in archive.infolist())
if shutil.disk_usage(SOURCE).free-unpacked < 160*1024**3:
raise RuntimeError('Insufficient safe disk reserve for Plus assets extraction')
status('extracting-assets',uncompressed_bytes=unpacked)
for info in archive.infolist():
destination=(SOURCE/'libero/libero'/info.filename).resolve()
if not destination.is_relative_to((SOURCE/'libero/libero').resolve()):raise ValueError('Unsafe zip member')
archive.extractall(SOURCE/'libero/libero')
assets=SOURCE/'libero/libero/assets'
if not assets.exists():
candidates=[p for p in (SOURCE/'libero/libero').rglob('assets') if p.is_dir()]
if len(candidates)!=1:raise RuntimeError('Ambiguous official asset archive layout')
assets.symlink_to(candidates[0],target_is_directory=True)
# Keep source asset SHA for a reproducible recovery; original remote remains available.
h=hashlib.sha256()
with ARCHIVE.open('rb') as f:
while chunk:=f.read(8*1024**2):h.update(chunk)
pipeline.write(LOCAL/'libero-plus-assets.json',dict(source='https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip',sha256=h.hexdigest(),bytes=ARCHIVE.stat().st_size,uncompressed_bytes=unpacked,benchmark_revision=REVISION))
ARCHIVE.unlink()
os.environ.update(ENV);sys.path.insert(0,str(SOURCE))
from libero.libero import benchmark
suite=benchmark.get_benchmark_dict()['libero_10']()
classification=read(SOURCE/'libero/libero/benchmark/task_classification.json')['libero_10']
by_name={r['name']:r for r in classification}; groups=collections.defaultdict(list)
for i in range(suite.n_tasks):
task=suite.get_task(i); meta=by_name[task.name]
groups[meta['category']].append(dict(task_id=i,init_state_id=0,seed=10000+i,task_name=task.name,category=meta['category'],difficulty=meta['difficulty_level']))
episodes=[]
for j in range(max(map(len,groups.values()))):
for category in sorted(groups):
if j<len(groups[category]):episodes.append(groups[category][j])
assert len(episodes)==2519
plan=dict(suite='libero_10',benchmark='LIBERO-Plus',benchmark_revision=REVISION,seed=10000,action_execution_horizon=10,solver_steps=10,settle_steps=10,max_steps=600,control_hz=20,protocol_note='One trial per official Plus-10 task; paired per-task environment/policy seed, 600 control steps, 10 settle steps. Not the four-suite leaderboard aggregate.',episodes=episodes)
plan_path=LOCAL/'libero-plus-plan.json'
if plan_path.exists():assert read(plan_path)==plan
else:pipeline.write(plan_path,plan)
for label in LABELS:
config=read(LOCAL/f'legacy-libero-{label}.json');config['paths']['libero_plus_source']=str(SOURCE);config['evaluation_profile']='matched_libero_plus_10'
path=LOCAL/f'libero-plus-{label}.json'
if path.exists():assert read(path)==config
else:pipeline.write(path,config)
backup()
# Validate all seven perturbation categories for all three policies first.
for label in LABELS:
out=LOCAL/'evaluations'/f'libero-plus-{label}'
if len(list(out.glob('task_*.json')))>=7:continue
status('seven-category-preflight',model=label)
if start(label,7).wait()!=0:raise RuntimeError('Seven-category preflight failed: '+label)
backup()
pending=list(LABELS);active={};last_backup=0
while pending or active:
while pending and len(active)<2:
label=pending.pop(0)
if read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json').get('complete'):continue
active[label]=start(label)
for label,child in list(active.items()):
code=child.poll()
if code is not None:
if code:raise RuntimeError('Evaluation failed: '+label)
del active[label]
counts={label:len(list((LOCAL/'evaluations'/f'libero-plus-{label}').glob('task_*.json'))) for label in LABELS}
status('evaluating',active={k:v.pid for k,v in active.items()},completed_episodes=counts,target_per_model=2519)
if time.time()-last_backup>300:
try:backup();last_backup=time.time()
except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
if pending or active:time.sleep(30)
result=dict(benchmark='LIBERO-Plus-10',plan=plan['protocol_note'],models={})
for label in LABELS:
s=read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json');assert s['complete'] and s['attempts']==2519
by_category={};by_difficulty={}
for r in s['episodes']:
for grouping,key in [(by_category,r['case']['category']),(by_difficulty,str(r['case']['difficulty']))]:
item=grouping.setdefault(key,dict(attempts=0,successes=0));item['attempts']+=1;item['successes']+=int(r['success'])
result['models'][label]=dict(attempts=s['attempts'],successes=s['successes'],success_rate=s['success_rate'],by_category=by_category,by_difficulty=by_difficulty)
pipeline.write(LOCAL/'libero-plus-results.json',result);status('completed')
backup(final=True)
notify('[LIBERO-Plus 완료] FT/L4/L18 각2519회 평가와 최종 HF readback 검증 완료. libero-plus-results.json 및 libero-plus-backup.json을 확인하고 전체/유형별 성공률을 보고하세요.')
if __name__=='__main__':
try:main()
except BlockingIOError:raise SystemExit('Plus driver already running')
except BaseException as error:
status('failed',error=type(error).__name__+': '+str(error))
try:backup();notify('[LIBERO-Plus 오류] libero-plus-status.json 및 audit/libero-plus-*.log를 확인해 복구하세요. 기존 프로세스부터 확인하고 중복 실행하지 마세요.')
except Exception:pass
raise