File size: 2,606 Bytes
c1f6b4d add1d5f c1f6b4d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | """Recover regenerable inputs after workspace restart, then continue the paired run."""
import json,os,pathlib,subprocess,time,hashlib
from datetime import datetime,timezone
ROOT=pathlib.Path(__file__).resolve().parents[1];LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi'
ENV={**os.environ,'CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))}
def status(stage,**more):
value=dict(stage=stage,state='running',pid=os.getpid(),updated_unix_time=time.time(),**more);p=LOCAL/'comparison-status.json';temp=p.with_suffix('.tmp');temp.write_text(json.dumps(value,indent=2));temp.replace(p);print(json.dumps(value),flush=True)
def run(name,args):
status(name)
with (LOCAL/'audit'/('restart-'+name+'.log')).open('a') as log:subprocess.run(list(map(str,args)),env=ENV,cwd=ROOT,stdout=log,stderr=subprocess.STDOUT,check=True)
def main():
os.chdir(ROOT);deadline=time.monotonic()+1800
data=LOCAL/'assets/processed-libero/asset-index.json'
while True:
try:
m=json.loads(data.read_text());ready=len(m.get('dev_ids',[]))==10 and all('file_sha256' in r for r in m['episodes'].values())
except (FileNotFoundError,json.JSONDecodeError):ready=False
if ready:break
if time.monotonic()>deadline:raise TimeoutError('Processed-data recovery did not complete; inspect restart-processed-data.log')
status('restoring_processed_data',main_updates_completed=0)
time.sleep(15)
run('cuda-smoke',[PYTHON,'-c',"import torch,json; x=torch.randn(512,512,device='cuda',dtype=torch.bfloat16); y=x@x.T; torch.cuda.synchronize(); assert bool(y.isfinite().all()); print(json.dumps(dict(torch=torch.__version__,device=torch.cuda.get_device_name(),bf16=torch.cuda.is_bf16_supported())))"])
run('simulator-smoke',[PYTHON,ROOT/'scripts/smoke_simulator.py'])
status('continuing_verified_comparison')
subprocess.run(['python3',str(ROOT/'scripts/launch_comparison.py'),'--continue-after-smoke','--on-demand'],env=ENV,cwd=ROOT,check=True)
if __name__=='__main__':
try:main()
except BaseException as error:
p=LOCAL/'restart-recovery-failure.json';p.write_text(json.dumps(dict(error=type(error).__name__+': '+str(error),updated_utc=datetime.now(timezone.utc).isoformat()),indent=2));raise
|