Download code/scripts/run_libero_plus.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/run_libero_plus.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/run_libero_plus.py
-
curl -L -o run_libero_plus.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/run_libero_plus.py
10.5 kB
| """Detached matched FT/L4/L18 LIBERO-Plus-10 evaluation, resumable by episode.""" | |
| import collections | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import subprocess | |
| import sys | |
| import time | |
| import zipfile | |
| import launch_comparison as pipeline | |
| ROOT, LOCAL = pipeline.ROOT, pipeline.LOCAL | |
| SOURCE = Path('/tmp/robottt-libero-plus') | |
| ARCHIVE = Path('/tmp/robottt-libero-plus-assets.zip') | |
| REVISION = '4976dc30028e805ff8094b55501d532c48fec182' | |
| ENV = {**pipeline.ENV, 'LIBERO_CONFIG_PATH': str(LOCAL/'libero-plus-config'), | |
| 'PYTHONPATH': str(SOURCE)+':'+pipeline.ENV['PYTHONPATH']} | |
| LABELS = ('Original','L4','L18') | |
| def read(path): | |
| return json.loads(path.read_text()) if path.exists() else {} | |
| def status(stage, **kw): | |
| pipeline.write(LOCAL/'libero-plus-status.json', dict(stage=stage,pid=os.getpid(),updated_unix_time=time.time(),**kw)) | |
| def notify(message): | |
| from run_legacy_libero import notify as send | |
| send(message) | |
| def backup(final=False): | |
| # Small incremental commits; the existing worker separately mirrors weights and code. | |
| from huggingface_hub import HfApi, CommitOperationAdd | |
| c=read(LOCAL/'hf-backup.json'); token=Path(c['token_file']).read_text().strip(); api=HfApi(token=token) | |
| files={} | |
| for path in LOCAL.glob('libero-plus-*.json'): files['progress/'+path.name]=path.read_bytes() | |
| for path in (ROOT/'scripts').glob('*libero_plus.py'): files['code/scripts/'+path.name]=path.read_bytes() | |
| for path in (LOCAL/'evaluations').glob('libero-plus-*/*.json'): files[str(path.relative_to(LOCAL))]=path.read_bytes() | |
| commit=api.create_commit(repo_id=c['repo_id'],operations=[CommitOperationAdd(path_in_repo=k,path_or_fileobj=v) for k,v in files.items()],commit_message='LIBERO-Plus matched evaluation '+('final' if final else 'progress')) | |
| if final: | |
| from huggingface_hub import hf_hub_download | |
| for name,expected in files.items(): | |
| actual=Path(hf_hub_download(c['repo_id'],name,revision=commit.oid,token=token,cache_dir='/tmp/robottt-plus-verify')).read_bytes() | |
| assert actual==expected,name | |
| pipeline.write(LOCAL/'libero-plus-backup.json',dict(commit_oid=commit.oid,files=len(files),remote_readback_verified=final,updated_unix_time=time.time())) | |
| def start(label, maximum=None): | |
| command=[str(pipeline.PYTHON),'scripts/evaluate_libero_plus.py','--run-config',str(LOCAL/f'libero-plus-{label}.json'),'--plan',str(LOCAL/'libero-plus-plan.json'),'--output',str(LOCAL/'evaluations'/f'libero-plus-{label}')] | |
| if maximum is not None:command+=['--max-episodes',str(maximum)] | |
| with (LOCAL/'audit'/f'libero-plus-{label}.log').open('a') as log: | |
| return subprocess.Popen(command,cwd=ROOT,env=ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT) | |
| def main(): | |
| lock=(LOCAL/'libero-plus-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| revision=subprocess.check_output(['git','-C',str(SOURCE),'rev-parse','HEAD'],text=True).strip() | |
| assert revision==REVISION | |
| if not (LOCAL/'libero-plus-assets.json').exists() or not (SOURCE/'libero/libero/assets').is_dir(): | |
| # Wait for the separately launched download to finish, including central directory. | |
| last_backup=0 | |
| while not zipfile.is_zipfile(ARCHIVE): | |
| # Adopt an existing download; only resume if no process still writes this file. | |
| writers=[] | |
| for proc in Path('/proc').glob('[0-9]*/cmdline'): | |
| try: | |
| args=proc.read_bytes().split(b'\0') | |
| if b'curl' in args and str(ARCHIVE).encode() in args:writers.append(int(proc.parent.name)) | |
| except (FileNotFoundError,PermissionError):pass | |
| if not writers: | |
| with Path('/tmp/robottt-plus-download.log').open('a') as log: | |
| child=subprocess.Popen(['curl','-fL','--retry','5','-C','-','https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip','-o',str(ARCHIVE)],stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) | |
| writers=[child.pid] | |
| status('downloading-assets',downloaded_bytes=ARCHIVE.stat().st_size if ARCHIVE.exists() else 0,download_pids=writers) | |
| if time.time()-last_backup>300: | |
| try:backup();last_backup=time.time() | |
| except Exception as error:print('Backup will retry:',type(error).__name__,flush=True) | |
| time.sleep(30) | |
| with zipfile.ZipFile(ARCHIVE) as archive: | |
| unpacked=sum(x.file_size for x in archive.infolist()) | |
| if shutil.disk_usage(SOURCE).free-unpacked < 160*1024**3: | |
| raise RuntimeError('Insufficient safe disk reserve for Plus assets extraction') | |
| status('extracting-assets',uncompressed_bytes=unpacked) | |
| for info in archive.infolist(): | |
| destination=(SOURCE/'libero/libero'/info.filename).resolve() | |
| if not destination.is_relative_to((SOURCE/'libero/libero').resolve()):raise ValueError('Unsafe zip member') | |
| archive.extractall(SOURCE/'libero/libero') | |
| assets=SOURCE/'libero/libero/assets' | |
| if not assets.exists(): | |
| candidates=[p for p in (SOURCE/'libero/libero').rglob('assets') if p.is_dir()] | |
| if len(candidates)!=1:raise RuntimeError('Ambiguous official asset archive layout') | |
| assets.symlink_to(candidates[0],target_is_directory=True) | |
| # Keep source asset SHA for a reproducible recovery; original remote remains available. | |
| h=hashlib.sha256() | |
| with ARCHIVE.open('rb') as f: | |
| while chunk:=f.read(8*1024**2):h.update(chunk) | |
| pipeline.write(LOCAL/'libero-plus-assets.json',dict(source='https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip',sha256=h.hexdigest(),bytes=ARCHIVE.stat().st_size,uncompressed_bytes=unpacked,benchmark_revision=REVISION)) | |
| ARCHIVE.unlink() | |
| os.environ.update(ENV);sys.path.insert(0,str(SOURCE)) | |
| from libero.libero import benchmark | |
| suite=benchmark.get_benchmark_dict()['libero_10']() | |
| classification=read(SOURCE/'libero/libero/benchmark/task_classification.json')['libero_10'] | |
| by_name={r['name']:r for r in classification}; groups=collections.defaultdict(list) | |
| for i in range(suite.n_tasks): | |
| task=suite.get_task(i); meta=by_name[task.name] | |
| groups[meta['category']].append(dict(task_id=i,init_state_id=0,seed=10000+i,task_name=task.name,category=meta['category'],difficulty=meta['difficulty_level'])) | |
| episodes=[] | |
| for j in range(max(map(len,groups.values()))): | |
| for category in sorted(groups): | |
| if j<len(groups[category]):episodes.append(groups[category][j]) | |
| assert len(episodes)==2519 | |
| plan=dict(suite='libero_10',benchmark='LIBERO-Plus',benchmark_revision=REVISION,seed=10000,action_execution_horizon=10,solver_steps=10,settle_steps=10,max_steps=600,control_hz=20,protocol_note='One trial per official Plus-10 task; paired per-task environment/policy seed, 600 control steps, 10 settle steps. Not the four-suite leaderboard aggregate.',episodes=episodes) | |
| plan_path=LOCAL/'libero-plus-plan.json' | |
| if plan_path.exists():assert read(plan_path)==plan | |
| else:pipeline.write(plan_path,plan) | |
| for label in LABELS: | |
| config=read(LOCAL/f'legacy-libero-{label}.json');config['paths']['libero_plus_source']=str(SOURCE);config['evaluation_profile']='matched_libero_plus_10' | |
| path=LOCAL/f'libero-plus-{label}.json' | |
| if path.exists():assert read(path)==config | |
| else:pipeline.write(path,config) | |
| backup() | |
| # Validate all seven perturbation categories for all three policies first. | |
| for label in LABELS: | |
| out=LOCAL/'evaluations'/f'libero-plus-{label}' | |
| if len(list(out.glob('task_*.json')))>=7:continue | |
| status('seven-category-preflight',model=label) | |
| if start(label,7).wait()!=0:raise RuntimeError('Seven-category preflight failed: '+label) | |
| backup() | |
| pending=list(LABELS);active={};last_backup=0 | |
| while pending or active: | |
| while pending and len(active)<2: | |
| label=pending.pop(0) | |
| if read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json').get('complete'):continue | |
| active[label]=start(label) | |
| for label,child in list(active.items()): | |
| code=child.poll() | |
| if code is not None: | |
| if code:raise RuntimeError('Evaluation failed: '+label) | |
| del active[label] | |
| counts={label:len(list((LOCAL/'evaluations'/f'libero-plus-{label}').glob('task_*.json'))) for label in LABELS} | |
| status('evaluating',active={k:v.pid for k,v in active.items()},completed_episodes=counts,target_per_model=2519) | |
| if time.time()-last_backup>300: | |
| try:backup();last_backup=time.time() | |
| except Exception as error:print('Backup will retry:',type(error).__name__,flush=True) | |
| if pending or active:time.sleep(30) | |
| result=dict(benchmark='LIBERO-Plus-10',plan=plan['protocol_note'],models={}) | |
| for label in LABELS: | |
| s=read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json');assert s['complete'] and s['attempts']==2519 | |
| by_category={};by_difficulty={} | |
| for r in s['episodes']: | |
| for grouping,key in [(by_category,r['case']['category']),(by_difficulty,str(r['case']['difficulty']))]: | |
| item=grouping.setdefault(key,dict(attempts=0,successes=0));item['attempts']+=1;item['successes']+=int(r['success']) | |
| result['models'][label]=dict(attempts=s['attempts'],successes=s['successes'],success_rate=s['success_rate'],by_category=by_category,by_difficulty=by_difficulty) | |
| pipeline.write(LOCAL/'libero-plus-results.json',result);status('completed') | |
| backup(final=True) | |
| notify('[LIBERO-Plus 완료] FT/L4/L18 각2519회 평가와 최종 HF readback 검증 완료. libero-plus-results.json 및 libero-plus-backup.json을 확인하고 전체/유형별 성공률을 보고하세요.') | |
| if __name__=='__main__': | |
| try:main() | |
| except BlockingIOError:raise SystemExit('Plus driver already running') | |
| except BaseException as error: | |
| status('failed',error=type(error).__name__+': '+str(error)) | |
| try:backup();notify('[LIBERO-Plus 오류] libero-plus-status.json 및 audit/libero-plus-*.log를 확인해 복구하세요. 기존 프로세스부터 확인하고 중복 실행하지 마세요.') | |
| except Exception:pass | |
| raise | |