"""Recover regenerable inputs after workspace restart, then continue the paired run.""" import json,os,pathlib,subprocess,time,hashlib from datetime import datetime,timezone ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi' ENV={**os.environ,'CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))} def status(stage,**more): value=dict(stage=stage,state='running',pid=os.getpid(),updated_unix_time=time.time(),**more);p=LOCAL/'comparison-status.json';temp=p.with_suffix('.tmp');temp.write_text(json.dumps(value,indent=2));temp.replace(p);print(json.dumps(value),flush=True) def run(name,args): status(name) with (LOCAL/'audit'/('restart-'+name+'.log')).open('a') as log:subprocess.run(list(map(str,args)),env=ENV,cwd=ROOT,stdout=log,stderr=subprocess.STDOUT,check=True) def main(): os.chdir(ROOT);deadline=time.monotonic()+1800 data=LOCAL/'assets/processed-libero/asset-index.json' while True: try: m=json.loads(data.read_text());ready=len(m.get('dev_ids',[]))==10 and all('file_sha256' in r for r in m['episodes'].values()) except (FileNotFoundError,json.JSONDecodeError):ready=False if ready:break if time.monotonic()>deadline:raise TimeoutError('Processed-data recovery did not complete; inspect restart-processed-data.log') status('restoring_processed_data',main_updates_completed=0) time.sleep(15) run('cuda-smoke',[PYTHON,'-c',"import torch,json; x=torch.randn(512,512,device='cuda',dtype=torch.bfloat16); y=x@x.T; torch.cuda.synchronize(); assert bool(y.isfinite().all()); print(json.dumps(dict(torch=torch.__version__,device=torch.cuda.get_device_name(),bf16=torch.cuda.is_bf16_supported())))"]) run('simulator-smoke',[PYTHON,ROOT/'scripts/smoke_simulator.py']) status('continuing_verified_comparison') subprocess.run(['python3',str(ROOT/'scripts/launch_comparison.py'),'--continue-after-smoke','--on-demand'],env=ENV,cwd=ROOT,check=True) if __name__=='__main__': try:main() except BaseException as error: p=LOCAL/'restart-recovery-failure.json';p.write_text(json.dumps(dict(error=type(error).__name__+': '+str(error),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2));raise