File size: 10,505 Bytes
240faaa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e947806
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112e766
 
 
 
 
e947806
 
 
 
 
 
240faaa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
"""Detached matched FT/L4/L18 LIBERO-Plus-10 evaluation, resumable by episode."""
import collections
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import zipfile
import launch_comparison as pipeline

ROOT, LOCAL = pipeline.ROOT, pipeline.LOCAL
SOURCE = Path('/tmp/robottt-libero-plus')
ARCHIVE = Path('/tmp/robottt-libero-plus-assets.zip')
REVISION = '4976dc30028e805ff8094b55501d532c48fec182'
ENV = {**pipeline.ENV, 'LIBERO_CONFIG_PATH': str(LOCAL/'libero-plus-config'),
       'PYTHONPATH': str(SOURCE)+':'+pipeline.ENV['PYTHONPATH']}
LABELS = ('Original','L4','L18')


def read(path):
    return json.loads(path.read_text()) if path.exists() else {}


def status(stage, **kw):
    pipeline.write(LOCAL/'libero-plus-status.json', dict(stage=stage,pid=os.getpid(),updated_unix_time=time.time(),**kw))


def notify(message):
    from run_legacy_libero import notify as send
    send(message)


def backup(final=False):
    # Small incremental commits; the existing worker separately mirrors weights and code.
    from huggingface_hub import HfApi, CommitOperationAdd
    c=read(LOCAL/'hf-backup.json'); token=Path(c['token_file']).read_text().strip(); api=HfApi(token=token)
    files={}
    for path in LOCAL.glob('libero-plus-*.json'): files['progress/'+path.name]=path.read_bytes()
    for path in (ROOT/'scripts').glob('*libero_plus.py'): files['code/scripts/'+path.name]=path.read_bytes()
    for path in (LOCAL/'evaluations').glob('libero-plus-*/*.json'): files[str(path.relative_to(LOCAL))]=path.read_bytes()
    commit=api.create_commit(repo_id=c['repo_id'],operations=[CommitOperationAdd(path_in_repo=k,path_or_fileobj=v) for k,v in files.items()],commit_message='LIBERO-Plus matched evaluation '+('final' if final else 'progress'))
    if final:
        from huggingface_hub import hf_hub_download
        for name,expected in files.items():
            actual=Path(hf_hub_download(c['repo_id'],name,revision=commit.oid,token=token,cache_dir='/tmp/robottt-plus-verify')).read_bytes()
            assert actual==expected,name
    pipeline.write(LOCAL/'libero-plus-backup.json',dict(commit_oid=commit.oid,files=len(files),remote_readback_verified=final,updated_unix_time=time.time()))


def start(label, maximum=None):
    command=[str(pipeline.PYTHON),'scripts/evaluate_libero_plus.py','--run-config',str(LOCAL/f'libero-plus-{label}.json'),'--plan',str(LOCAL/'libero-plus-plan.json'),'--output',str(LOCAL/'evaluations'/f'libero-plus-{label}')]
    if maximum is not None:command+=['--max-episodes',str(maximum)]
    with (LOCAL/'audit'/f'libero-plus-{label}.log').open('a') as log:
        return subprocess.Popen(command,cwd=ROOT,env=ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT)


def main():
    lock=(LOCAL/'libero-plus-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    revision=subprocess.check_output(['git','-C',str(SOURCE),'rev-parse','HEAD'],text=True).strip()
    assert revision==REVISION
    if not (LOCAL/'libero-plus-assets.json').exists() or not (SOURCE/'libero/libero/assets').is_dir():
        # Wait for the separately launched download to finish, including central directory.
        last_backup=0
        while not zipfile.is_zipfile(ARCHIVE):
            # Adopt an existing download; only resume if no process still writes this file.
            writers=[]
            for proc in Path('/proc').glob('[0-9]*/cmdline'):
                try:
                    args=proc.read_bytes().split(b'\0')
                    if b'curl' in args and str(ARCHIVE).encode() in args:writers.append(int(proc.parent.name))
                except (FileNotFoundError,PermissionError):pass
            if not writers:
                with Path('/tmp/robottt-plus-download.log').open('a') as log:
                    child=subprocess.Popen(['curl','-fL','--retry','5','-C','-','https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip','-o',str(ARCHIVE)],stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
                writers=[child.pid]
            status('downloading-assets',downloaded_bytes=ARCHIVE.stat().st_size if ARCHIVE.exists() else 0,download_pids=writers)
            if time.time()-last_backup>300:
                try:backup();last_backup=time.time()
                except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
            time.sleep(30)
        with zipfile.ZipFile(ARCHIVE) as archive:
            unpacked=sum(x.file_size for x in archive.infolist())
            if shutil.disk_usage(SOURCE).free-unpacked < 160*1024**3:
                raise RuntimeError('Insufficient safe disk reserve for Plus assets extraction')
            status('extracting-assets',uncompressed_bytes=unpacked)
            for info in archive.infolist():
                destination=(SOURCE/'libero/libero'/info.filename).resolve()
                if not destination.is_relative_to((SOURCE/'libero/libero').resolve()):raise ValueError('Unsafe zip member')
            archive.extractall(SOURCE/'libero/libero')
        assets=SOURCE/'libero/libero/assets'
        if not assets.exists():
            candidates=[p for p in (SOURCE/'libero/libero').rglob('assets') if p.is_dir()]
            if len(candidates)!=1:raise RuntimeError('Ambiguous official asset archive layout')
            assets.symlink_to(candidates[0],target_is_directory=True)
        # Keep source asset SHA for a reproducible recovery; original remote remains available.
        h=hashlib.sha256()
        with ARCHIVE.open('rb') as f:
            while chunk:=f.read(8*1024**2):h.update(chunk)
        pipeline.write(LOCAL/'libero-plus-assets.json',dict(source='https://huggingface.co/datasets/Sylvest/LIBERO-plus/resolve/main/assets.zip',sha256=h.hexdigest(),bytes=ARCHIVE.stat().st_size,uncompressed_bytes=unpacked,benchmark_revision=REVISION))
        ARCHIVE.unlink()
    os.environ.update(ENV);sys.path.insert(0,str(SOURCE))
    from libero.libero import benchmark
    suite=benchmark.get_benchmark_dict()['libero_10']()
    classification=read(SOURCE/'libero/libero/benchmark/task_classification.json')['libero_10']
    by_name={r['name']:r for r in classification}; groups=collections.defaultdict(list)
    for i in range(suite.n_tasks):
        task=suite.get_task(i); meta=by_name[task.name]
        groups[meta['category']].append(dict(task_id=i,init_state_id=0,seed=10000+i,task_name=task.name,category=meta['category'],difficulty=meta['difficulty_level']))
    episodes=[]
    for j in range(max(map(len,groups.values()))):
        for category in sorted(groups):
            if j<len(groups[category]):episodes.append(groups[category][j])
    assert len(episodes)==2519
    plan=dict(suite='libero_10',benchmark='LIBERO-Plus',benchmark_revision=REVISION,seed=10000,action_execution_horizon=10,solver_steps=10,settle_steps=10,max_steps=600,control_hz=20,protocol_note='One trial per official Plus-10 task; paired per-task environment/policy seed, 600 control steps, 10 settle steps. Not the four-suite leaderboard aggregate.',episodes=episodes)
    plan_path=LOCAL/'libero-plus-plan.json'
    if plan_path.exists():assert read(plan_path)==plan
    else:pipeline.write(plan_path,plan)
    for label in LABELS:
        config=read(LOCAL/f'legacy-libero-{label}.json');config['paths']['libero_plus_source']=str(SOURCE);config['evaluation_profile']='matched_libero_plus_10'
        path=LOCAL/f'libero-plus-{label}.json'
        if path.exists():assert read(path)==config
        else:pipeline.write(path,config)
    backup()
    # Validate all seven perturbation categories for all three policies first.
    for label in LABELS:
        out=LOCAL/'evaluations'/f'libero-plus-{label}'
        if len(list(out.glob('task_*.json')))>=7:continue
        status('seven-category-preflight',model=label)
        if start(label,7).wait()!=0:raise RuntimeError('Seven-category preflight failed: '+label)
        backup()
    pending=list(LABELS);active={};last_backup=0
    while pending or active:
        while pending and len(active)<2:
            label=pending.pop(0)
            if read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json').get('complete'):continue
            active[label]=start(label)
        for label,child in list(active.items()):
            code=child.poll()
            if code is not None:
                if code:raise RuntimeError('Evaluation failed: '+label)
                del active[label]
        counts={label:len(list((LOCAL/'evaluations'/f'libero-plus-{label}').glob('task_*.json'))) for label in LABELS}
        status('evaluating',active={k:v.pid for k,v in active.items()},completed_episodes=counts,target_per_model=2519)
        if time.time()-last_backup>300:
            try:backup();last_backup=time.time()
            except Exception as error:print('Backup will retry:',type(error).__name__,flush=True)
        if pending or active:time.sleep(30)
    result=dict(benchmark='LIBERO-Plus-10',plan=plan['protocol_note'],models={})
    for label in LABELS:
        s=read(LOCAL/'evaluations'/f'libero-plus-{label}'/'summary.json');assert s['complete'] and s['attempts']==2519
        by_category={};by_difficulty={}
        for r in s['episodes']:
            for grouping,key in [(by_category,r['case']['category']),(by_difficulty,str(r['case']['difficulty']))]:
                item=grouping.setdefault(key,dict(attempts=0,successes=0));item['attempts']+=1;item['successes']+=int(r['success'])
        result['models'][label]=dict(attempts=s['attempts'],successes=s['successes'],success_rate=s['success_rate'],by_category=by_category,by_difficulty=by_difficulty)
    pipeline.write(LOCAL/'libero-plus-results.json',result);status('completed')
    backup(final=True)
    notify('[LIBERO-Plus 완료] FT/L4/L18 각2519회 평가와 최종 HF readback 검증 완료. libero-plus-results.json 및 libero-plus-backup.json을 확인하고 전체/유형별 성공률을 보고하세요.')


if __name__=='__main__':
    try:main()
    except BlockingIOError:raise SystemExit('Plus driver already running')
    except BaseException as error:
        status('failed',error=type(error).__name__+': '+str(error))
        try:backup();notify('[LIBERO-Plus 오류] libero-plus-status.json 및 audit/libero-plus-*.log를 확인해 복구하세요. 기존 프로세스부터 확인하고 중복 실행하지 마세요.')
        except Exception:pass
        raise