File size: 2,606 Bytes
c1f6b4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
add1d5f
c1f6b4d
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
"""Recover regenerable inputs after workspace restart, then continue the paired run."""
import json,os,pathlib,subprocess,time,hashlib
from datetime import datetime,timezone
ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi'
ENV={**os.environ,'CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))}

def status(stage,**more):
    value=dict(stage=stage,state='running',pid=os.getpid(),updated_unix_time=time.time(),**more);p=LOCAL/'comparison-status.json';temp=p.with_suffix('.tmp');temp.write_text(json.dumps(value,indent=2));temp.replace(p);print(json.dumps(value),flush=True)

def run(name,args):
    status(name)
    with (LOCAL/'audit'/('restart-'+name+'.log')).open('a') as log:subprocess.run(list(map(str,args)),env=ENV,cwd=ROOT,stdout=log,stderr=subprocess.STDOUT,check=True)

def main():
    os.chdir(ROOT);deadline=time.monotonic()+1800
    data=LOCAL/'assets/processed-libero/asset-index.json'
    while True:
        try:
            m=json.loads(data.read_text());ready=len(m.get('dev_ids',[]))==10 and all('file_sha256' in r for r in m['episodes'].values())
        except (FileNotFoundError,json.JSONDecodeError):ready=False
        if ready:break
        if time.monotonic()>deadline:raise TimeoutError('Processed-data recovery did not complete; inspect restart-processed-data.log')
        status('restoring_processed_data',main_updates_completed=0)
        time.sleep(15)
    run('cuda-smoke',[PYTHON,'-c',"import torch,json; x=torch.randn(512,512,device='cuda',dtype=torch.bfloat16); y=x@x.T; torch.cuda.synchronize(); assert bool(y.isfinite().all()); print(json.dumps(dict(torch=torch.__version__,device=torch.cuda.get_device_name(),bf16=torch.cuda.is_bf16_supported())))"])
    run('simulator-smoke',[PYTHON,ROOT/'scripts/smoke_simulator.py'])
    status('continuing_verified_comparison')
    subprocess.run(['python3',str(ROOT/'scripts/launch_comparison.py'),'--continue-after-smoke','--on-demand'],env=ENV,cwd=ROOT,check=True)

if __name__=='__main__':
    try:main()
    except BaseException as error:
        p=LOCAL/'restart-recovery-failure.json';p.write_text(json.dumps(dict(error=type(error).__name__+': '+str(error),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2));raise