File size: 10,505 Bytes
240faaa e947806 112e766 e947806 240faaa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 | """Detached matched FT/L4/L18 LIBERO-Plus-10 evaluation, resumable by episode."""
import collections
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import zipfile
import launch_comparison as pipeline
ROOT, LOCAL = pipeline.ROOT, pipeline.LOCAL
SOURCE = Path('/tmp/robottt-libero-plus')
ARCHIVE = Path('/tmp/robottt-libero-plus-assets.zip')
REVISION = '4976dc30028e805ff8094b55501d532c48fec182'
ENV = {**pipeline.ENV, 'LIBERO_CONFIG_PATH': str(LOCAL/'libero-plus-config'),
'PYTHONPATH': str(SOURCE)+':'+pipeline.ENV['PYTHONPATH']}
LABELS = ('Original','L4','L18')
def read(path):
return json.loads(path.read_text()) if path.exists() else {}
def status(stage, **kw):
pipeline.write(LOCAL/'libero-plus-status.json', dict(stage=stage,pid=os.getpid(),updated_unix_time=time.time(),**kw))
def notify(message):
from run_legacy_libero import notify as send
send(message)
def backup(final=False):
# Small incremental commits; the existing worker separately mirrors weights and code.
from huggingface_hub import HfApi, CommitOperationAdd
c=read(LOCAL/'hf-backup.json'); token=Path(c['token_file']).read_text().strip(); api=HfApi(token=token)
files={}
for path in LOCAL.glob('libero-plus-*.json'): files['progress/'+path.name]=path.read_bytes()
for path in (ROOT/'scripts').glob('*libero_plus.py'): files['code/scripts/'+path.name]=path.read_bytes()
for path in (LOCAL/'evaluations').glob('libero-plus-*/*.json'): files[str(path.relative_to(LOCAL))]=path.read_bytes()
commit=api.create_commit(repo_id=c['repo_id'],operations=[CommitOperationAdd(path_in_repo=k,path_or_fileobj=v) for k,v in files.items()],commit_message='LIBERO-Plus matched evaluation '+('final' if final else 'progress'))
if final:
from huggingface_hub import hf_hub_download
for name,expected in files.items():
actual=Path(hf_hub_download(c['repo_id'],name,revision=commit.oid,token=token,cache_dir='/tmp/robottt-plus-verify')).read_bytes()
assert actual==expected,name
pipeline.write(LOCAL/'libero-plus-backup.json',dict(commit_oid=commit.oid,files=len(files),remote_readback_verified=final,updated_unix_time=time.time()))
def start(label, maximum=None):
command=[str(pipeline.PYTHON),'scripts/evaluate_libero_plus.py','--run-config',str(LOCAL/f'libero-plus-{label}.json'),'--plan',str(LOCAL/'libero-plus-plan.json'),'--output',str(LOCAL/'evaluations'/f'libero-plus-{label}')]
if maximum is not None:command+=['--max-episodes',str(maximum)]
with (LOCAL/'audit'/f'libero-plus-{label}.log').open('a') as log:
return subprocess.Popen(command,cwd=ROOT,env=ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT)
def main():
lock=(LOCAL/'libero-plus-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
revision=subprocess.check_output(['git','-C',str(SOURCE),'rev-parse','HEAD'],text=True).strip()
assert revision==REVISION
if not (LOCAL/'libero-plus-assets.json').exists() or not (SOURCE/'libero/libero/assets').is_dir():
# Wait for the separately launched download to finish, including central directory.
last_backup=0
while not zipfile.is_zipfile(ARCHIVE):
# Adopt an existing download; only resume if no process still writes this file.
writers=[]
for proc in Path('/proc').glob('[0-9]*/cmdline'):
try:
args=proc.read_bytes().split(b'\0')
if b'curl' in args and str(ARCHIVE).encode() in args:writers.append(int(proc.parent.name))
except (FileNotFoundError,PermissionError):pass
if not writers:
with Path('/tmp/robottt-plus-download.log').open('a') as log:
child=subprocess.Popen(['curl','-fL','--retry','5','-C','-','https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip','-o',str(ARCHIVE)],stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
writers=[child.pid]
status('downloading-assets',downloaded_bytes=ARCHIVE.stat().st_size if ARCHIVE.exists() else 0,download_pids=writers)
if time.time()-last_backup>300:
try:backup();last_backup=time.time()
except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
time.sleep(30)
with zipfile.ZipFile(ARCHIVE) as archive:
unpacked=sum(x.file_size for x in archive.infolist())
if shutil.disk_usage(SOURCE).free-unpacked < 160*1024**3:
raise RuntimeError('Insufficient safe disk reserve for Plus assets extraction')
status('extracting-assets',uncompressed_bytes=unpacked)
for info in archive.infolist():
destination=(SOURCE/'libero/libero'/info.filename).resolve()
if not destination.is_relative_to((SOURCE/'libero/libero').resolve()):raise ValueError('Unsafe zip member')
archive.extractall(SOURCE/'libero/libero')
assets=SOURCE/'libero/libero/assets'
if not assets.exists():
candidates=[p for p in (SOURCE/'libero/libero').rglob('assets') if p.is_dir()]
if len(candidates)!=1:raise RuntimeError('Ambiguous official asset archive layout')
assets.symlink_to(candidates[0],target_is_directory=True)
# Keep source asset SHA for a reproducible recovery; original remote remains available.
h=hashlib.sha256()
with ARCHIVE.open('rb') as f:
while chunk:=f.read(8*1024**2):h.update(chunk)
pipeline.write(LOCAL/'libero-plus-assets.json',dict(source='https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip',sha256=h.hexdigest(),bytes=ARCHIVE.stat().st_size,uncompressed_bytes=unpacked,benchmark_revision=REVISION))
ARCHIVE.unlink()
os.environ.update(ENV);sys.path.insert(0,str(SOURCE))
from libero.libero import benchmark
suite=benchmark.get_benchmark_dict()['libero_10']()
classification=read(SOURCE/'libero/libero/benchmark/task_classification.json')['libero_10']
by_name={r['name']:r for r in classification}; groups=collections.defaultdict(list)
for i in range(suite.n_tasks):
task=suite.get_task(i); meta=by_name[task.name]
groups[meta['category']].append(dict(task_id=i,init_state_id=0,seed=10000+i,task_name=task.name,category=meta['category'],difficulty=meta['difficulty_level']))
episodes=[]
for j in range(max(map(len,groups.values()))):
for category in sorted(groups):
if j<len(groups[category]):episodes.append(groups[category][j])
assert len(episodes)==2519
plan=dict(suite='libero_10',benchmark='LIBERO-Plus',benchmark_revision=REVISION,seed=10000,action_execution_horizon=10,solver_steps=10,settle_steps=10,max_steps=600,control_hz=20,protocol_note='One trial per official Plus-10 task; paired per-task environment/policy seed, 600 control steps, 10 settle steps. Not the four-suite leaderboard aggregate.',episodes=episodes)
plan_path=LOCAL/'libero-plus-plan.json'
if plan_path.exists():assert read(plan_path)==plan
else:pipeline.write(plan_path,plan)
for label in LABELS:
config=read(LOCAL/f'legacy-libero-{label}.json');config['paths']['libero_plus_source']=str(SOURCE);config['evaluation_profile']='matched_libero_plus_10'
path=LOCAL/f'libero-plus-{label}.json'
if path.exists():assert read(path)==config
else:pipeline.write(path,config)
backup()
# Validate all seven perturbation categories for all three policies first.
for label in LABELS:
out=LOCAL/'evaluations'/f'libero-plus-{label}'
if len(list(out.glob('task_*.json')))>=7:continue
status('seven-category-preflight',model=label)
if start(label,7).wait()!=0:raise RuntimeError('Seven-category preflight failed: '+label)
backup()
pending=list(LABELS);active={};last_backup=0
while pending or active:
while pending and len(active)<2:
label=pending.pop(0)
if read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json').get('complete'):continue
active[label]=start(label)
for label,child in list(active.items()):
code=child.poll()
if code is not None:
if code:raise RuntimeError('Evaluation failed: '+label)
del active[label]
counts={label:len(list((LOCAL/'evaluations'/f'libero-plus-{label}').glob('task_*.json'))) for label in LABELS}
status('evaluating',active={k:v.pid for k,v in active.items()},completed_episodes=counts,target_per_model=2519)
if time.time()-last_backup>300:
try:backup();last_backup=time.time()
except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
if pending or active:time.sleep(30)
result=dict(benchmark='LIBERO-Plus-10',plan=plan['protocol_note'],models={})
for label in LABELS:
s=read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json');assert s['complete'] and s['attempts']==2519
by_category={};by_difficulty={}
for r in s['episodes']:
for grouping,key in [(by_category,r['case']['category']),(by_difficulty,str(r['case']['difficulty']))]:
item=grouping.setdefault(key,dict(attempts=0,successes=0));item['attempts']+=1;item['successes']+=int(r['success'])
result['models'][label]=dict(attempts=s['attempts'],successes=s['successes'],success_rate=s['success_rate'],by_category=by_category,by_difficulty=by_difficulty)
pipeline.write(LOCAL/'libero-plus-results.json',result);status('completed')
backup(final=True)
notify('[LIBERO-Plus 완료] FT/L4/L18 각2519회 평가와 최종 HF readback 검증 완료. libero-plus-results.json 및 libero-plus-backup.json을 확인하고 전체/유형별 성공률을 보고하세요.')
if __name__=='__main__':
try:main()
except BlockingIOError:raise SystemExit('Plus driver already running')
except BaseException as error:
status('failed',error=type(error).__name__+': '+str(error))
try:backup();notify('[LIBERO-Plus 오류] libero-plus-status.json 및 audit/libero-plus-*.log를 확인해 복구하세요. 기존 프로세스부터 확인하고 중복 실행하지 마세요.')
except Exception:pass
raise
|