"""Detached matched FT/L4/L18 LIBERO-Plus-10 evaluation, resumable by episode.""" import collections import fcntl import hashlib import json import os from pathlib import Path import shutil import subprocess import sys import time import zipfile import launch_comparison as pipeline ROOT, LOCAL = pipeline.ROOT, pipeline.LOCAL SOURCE = Path('/tmp/robottt-libero-plus') ARCHIVE = Path('/tmp/robottt-libero-plus-assets.zip') REVISION = '4976dc30028e805ff8094b55501d532c48fec182' ENV = {**pipeline.ENV, 'LIBERO_CONFIG_PATH': str(LOCAL/'libero-plus-config'), 'PYTHONPATH': str(SOURCE)+':'+pipeline.ENV['PYTHONPATH']} LABELS = ('Original','L4','L18') def read(path): return json.loads(path.read_text()) if path.exists() else {} def status(stage, **kw): pipeline.write(LOCAL/'libero-plus-status.json', dict(stage=stage,pid=os.getpid(),updated_unix_time=time.time(),**kw)) def notify(message): from run_legacy_libero import notify as send send(message) def backup(final=False): # Small incremental commits; the existing worker separately mirrors weights and code. from huggingface_hub import HfApi, CommitOperationAdd c=read(LOCAL/'hf-backup.json'); token=Path(c['token_file']).read_text().strip(); api=HfApi(token=token) files={} for path in LOCAL.glob('libero-plus-*.json'): files['progress/'+path.name]=path.read_bytes() for path in (ROOT/'scripts').glob('*libero_plus.py'): files['code/scripts/'+path.name]=path.read_bytes() for path in (LOCAL/'evaluations').glob('libero-plus-*/*.json'): files[str(path.relative_to(LOCAL))]=path.read_bytes() commit=api.create_commit(repo_id=c['repo_id'],operations=[CommitOperationAdd(path_in_repo=k,path_or_fileobj=v) for k,v in files.items()],commit_message='LIBERO-Plus matched evaluation '+('final' if final else 'progress')) if final: from huggingface_hub import hf_hub_download for name,expected in files.items(): actual=Path(hf_hub_download(c['repo_id'],name,revision=commit.oid,token=token,cache_dir='/tmp/robottt-plus-verify')).read_bytes() assert actual==expected,name pipeline.write(LOCAL/'libero-plus-backup.json',dict(commit_oid=commit.oid,files=len(files),remote_readback_verified=final,updated_unix_time=time.time())) def start(label, maximum=None): command=[str(pipeline.PYTHON),'scripts/evaluate_libero_plus.py','--run-config',str(LOCAL/f'libero-plus-{label}.json'),'--plan',str(LOCAL/'libero-plus-plan.json'),'--output',str(LOCAL/'evaluations'/f'libero-plus-{label}')] if maximum is not None:command+=['--max-episodes',str(maximum)] with (LOCAL/'audit'/f'libero-plus-{label}.log').open('a') as log: return subprocess.Popen(command,cwd=ROOT,env=ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT) def main(): lock=(LOCAL/'libero-plus-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) revision=subprocess.check_output(['git','-C',str(SOURCE),'rev-parse','HEAD'],text=True).strip() assert revision==REVISION if not (LOCAL/'libero-plus-assets.json').exists() or not (SOURCE/'libero/libero/assets').is_dir(): # Wait for the separately launched download to finish, including central directory. last_backup=0 while not zipfile.is_zipfile(ARCHIVE): # Adopt an existing download; only resume if no process still writes this file. writers=[] for proc in Path('/proc').glob('[0-9]*/cmdline'): try: args=proc.read_bytes().split(b'\0') if b'curl' in args and str(ARCHIVE).encode() in args:writers.append(int(proc.parent.name)) except (FileNotFoundError,PermissionError):pass if not writers: with Path('/tmp/robottt-plus-download.log').open('a') as log: child=subprocess.Popen(['curl','-fL','--retry','5','-C','-','https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip','-o',str(ARCHIVE)],stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) writers=[child.pid] status('downloading-assets',downloaded_bytes=ARCHIVE.stat().st_size if ARCHIVE.exists() else 0,download_pids=writers) if time.time()-last_backup>300: try:backup();last_backup=time.time() except Exception as error:print('Backup will retry:',type(error).__name__,flush=True) time.sleep(30) with zipfile.ZipFile(ARCHIVE) as archive: unpacked=sum(x.file_size for x in archive.infolist()) if shutil.disk_usage(SOURCE).free-unpacked < 160*1024**3: raise RuntimeError('Insufficient safe disk reserve for Plus assets extraction') status('extracting-assets',uncompressed_bytes=unpacked) for info in archive.infolist(): destination=(SOURCE/'libero/libero'/info.filename).resolve() if not destination.is_relative_to((SOURCE/'libero/libero').resolve()):raise ValueError('Unsafe zip member') archive.extractall(SOURCE/'libero/libero') assets=SOURCE/'libero/libero/assets' if not assets.exists(): candidates=[p for p in (SOURCE/'libero/libero').rglob('assets') if p.is_dir()] if len(candidates)!=1:raise RuntimeError('Ambiguous official asset archive layout') assets.symlink_to(candidates[0],target_is_directory=True) # Keep source asset SHA for a reproducible recovery; original remote remains available. h=hashlib.sha256() with ARCHIVE.open('rb') as f: while chunk:=f.read(8*1024**2):h.update(chunk) pipeline.write(LOCAL/'libero-plus-assets.json',dict(source='https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip',sha256=h.hexdigest(),bytes=ARCHIVE.stat().st_size,uncompressed_bytes=unpacked,benchmark_revision=REVISION)) ARCHIVE.unlink() os.environ.update(ENV);sys.path.insert(0,str(SOURCE)) from libero.libero import benchmark suite=benchmark.get_benchmark_dict()['libero_10']() classification=read(SOURCE/'libero/libero/benchmark/task_classification.json')['libero_10'] by_name={r['name']:r for r in classification}; groups=collections.defaultdict(list) for i in range(suite.n_tasks): task=suite.get_task(i); meta=by_name[task.name] groups[meta['category']].append(dict(task_id=i,init_state_id=0,seed=10000+i,task_name=task.name,category=meta['category'],difficulty=meta['difficulty_level'])) episodes=[] for j in range(max(map(len,groups.values()))): for category in sorted(groups): if j=7:continue status('seven-category-preflight',model=label) if start(label,7).wait()!=0:raise RuntimeError('Seven-category preflight failed: '+label) backup() pending=list(LABELS);active={};last_backup=0 while pending or active: while pending and len(active)<2: label=pending.pop(0) if read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json').get('complete'):continue active[label]=start(label) for label,child in list(active.items()): code=child.poll() if code is not None: if code:raise RuntimeError('Evaluation failed: '+label) del active[label] counts={label:len(list((LOCAL/'evaluations'/f'libero-plus-{label}').glob('task_*.json'))) for label in LABELS} status('evaluating',active={k:v.pid for k,v in active.items()},completed_episodes=counts,target_per_model=2519) if time.time()-last_backup>300: try:backup();last_backup=time.time() except Exception as error:print('Backup will retry:',type(error).__name__,flush=True) if pending or active:time.sleep(30) result=dict(benchmark='LIBERO-Plus-10',plan=plan['protocol_note'],models={}) for label in LABELS: s=read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json');assert s['complete'] and s['attempts']==2519 by_category={};by_difficulty={} for r in s['episodes']: for grouping,key in [(by_category,r['case']['category']),(by_difficulty,str(r['case']['difficulty']))]: item=grouping.setdefault(key,dict(attempts=0,successes=0));item['attempts']+=1;item['successes']+=int(r['success']) result['models'][label]=dict(attempts=s['attempts'],successes=s['successes'],success_rate=s['success_rate'],by_category=by_category,by_difficulty=by_difficulty) pipeline.write(LOCAL/'libero-plus-results.json',result);status('completed') backup(final=True) notify('[LIBERO-Plus 완료] FT/L4/L18 각2519회 평가와 최종 HF readback 검증 완료. libero-plus-results.json 및 libero-plus-backup.json을 확인하고 전체/유형별 성공률을 보고하세요.') if __name__=='__main__': try:main() except BlockingIOError:raise SystemExit('Plus driver already running') except BaseException as error: status('failed',error=type(error).__name__+': '+str(error)) try:backup();notify('[LIBERO-Plus 오류] libero-plus-status.json 및 audit/libero-plus-*.log를 확인해 복구하세요. 기존 프로세스부터 확인하고 중복 실행하지 마세요.') except Exception:pass raise