"""Resume the persistent experiment after workspace interruption. Run: python3 scripts/resume_comparison.py Never restarts a live driver or automatically retries a recorded training error. """ import ctypes.util,fcntl,json,os,pathlib,shutil,subprocess,sys,time import launch_comparison as pipeline from workspace_storage import activate ROOT=pipeline.ROOT;LOCAL=pipeline.LOCAL def restore_system_libraries(): if ctypes.util.find_library('GL') and ctypes.util.find_library('OSMesa'):return packages=sorted((LOCAL/'system-packages').glob('*.deb')) if not packages:raise RuntimeError('Persistent renderer packages missing; restore system libraries first') pipeline.run('restore-system-libraries',['dpkg','-i',*packages]) def restore_python_runtime(): if pipeline.PYTHON.is_file():return archive=LOCAL/'runtime-env.tar.gz' if not archive.exists(): pipeline.run('download-python-runtime',[LOCAL/'hf-backup-venv/bin/python',ROOT/'scripts/download_runtime_archive.py']) archive=pathlib.Path('/tmp/robottt-runtime-download/environment/runtime-env.tar.gz') pipeline.run('restore-python-runtime',[sys.executable,ROOT/'scripts/archive_runtime.py','restore','--archive',archive,'--output','/tmp/robottt-runtime/venv']) for alias,target in [(LOCAL/'venv-persistent',pathlib.Path('/tmp/robottt-runtime/venv')),(LOCAL/'venv',LOCAL/'venv-persistent')]: if not alias.exists() and not alias.is_symlink():alias.symlink_to(target,target_is_directory=True) def workers(): for name,python,args in [('monitor',sys.executable,['scripts/monitor_comparison.py','--watch']),('hf-backup',str(LOCAL/'hf-backup-venv/bin/python'),['scripts/backup_huggingface.py','--watch'])]: pidfile=LOCAL/(name+'-worker.pid') if pidfile.exists(): command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' if command.exists() and args[0].encode() in command.read_bytes():continue with (LOCAL/'audit'/(name+'-worker.log')).open('a') as log: process=subprocess.Popen([python,*args],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) pidfile.write_text(str(process.pid)) if (LOCAL/'completion-watch.json').exists(): pidfile=LOCAL/'completion-watch-worker.pid' command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' if pidfile.exists() else None if command is None or not command.exists() or b'watch_comparison.py' not in command.read_bytes(): with (LOCAL/'audit/completion-watch-worker.log').open('a') as log: process=subprocess.Popen([sys.executable,'scripts/watch_comparison.py','--watch'],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) pidfile.write_text(str(process.pid)) def main(): os.chdir(ROOT) activated=activate() lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) support=LOCAL/'runtime-support.tar.gz' if support.exists():pipeline.run('restore-runtime-support',[sys.executable,ROOT/'scripts/archive_runtime.py','restore-support','--archive',support]) restore_python_runtime() if activated:pipeline.run('restore-remote-progress',[pipeline.PYTHON,ROOT/'scripts/restore_remote_progress.py']) workers();restore_system_libraries() gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name','--format=csv,noheader'],text=True).strip() uuid,name=[value.strip() for value in gpu.split(',')] if name!='NVIDIA GeForce RTX 3090':raise RuntimeError('Replacement GPU model requires validation') pipeline.write(LOCAL/'audit'/f'resume-hardware-{time.time_ns()}.json',dict(gpu=gpu,updated_unix_time=time.time())) runs=json.loads((LOCAL/'common-artifacts.json').read_text())['run_configs'] for kind,config in runs.items(): rp=pathlib.Path(config);r=json.loads(rp.read_text());out=pathlib.Path(r['paths']['output']) if (out/'completed.json').exists(): pipeline.run('verify-completed-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume']) elif (out/'checkpoints/latest.pt').exists(): pipeline.run('recover-saved-'+kind,[pipeline.PYTHON,ROOT/'scripts/recover_checkpoint.py','--run-config',rp]) pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume']) else: out.mkdir(parents=True,exist_ok=True) with (out/'run.lock').open('a') as run_lock: fcntl.flock(run_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if list(out.glob('failure*.json')):raise RuntimeError('Recorded failure needs diagnosis') journal=out/'ledger/events.jsonl' if journal.exists() and journal.stat().st_size: rows=[json.loads(line) for line in journal.read_text().splitlines()] if any(row.get('kind')=='optimizer_update' for row in rows):raise RuntimeError('Missing checkpoint after optimizer attempts; restore it from HF') archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True) shutil.copyfile(rp,archive/'config.json');shutil.move(journal,archive/'events.jsonl') r['gpu_uuid']=uuid;r['limits']['model_load']=1024;pipeline.write(rp,r) pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp]) flow=LOCAL/'evaluations'/('flow-'+kind);summary=flow/'summary.json' if not summary.exists(): if flow.exists(): archive=LOCAL/'evaluations/interruption-archives';archive.mkdir(parents=True,exist_ok=True) shutil.move(flow,archive/(flow.name+'-'+str(time.time_ns()))) pipeline.run('flow-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',flow]) elif json.loads(summary.read_text())['run_config_sha256']!=pipeline.digest(rp):raise ValueError('Completed flow evaluation identity differs') for kind,config in runs.items(): output=LOCAL/'evaluations'/('libero-'+kind);summary=output/'summary.json' if summary.exists() and json.loads(summary.read_text()).get('complete'):continue pipeline.run('libero-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',config,'--plan',LOCAL/'libero-eval-plan.json','--output',output]) pipeline.run('summarize',[pipeline.PYTHON,ROOT/'scripts/summarize_comparison.py']) pipeline.stage('completed','completed') if __name__=='__main__': try:main() except BlockingIOError:raise SystemExit('An experiment driver is already active; no changes made.') except BaseException as error: pipeline.stage('recovery_failed','failed',error=type(error).__name__+': '+str(error));raise