Download code/scripts/resume_comparison.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 6.95 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/resume_comparison.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/scripts/resume_comparison.py
-
curl -L -o resume_comparison.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/scripts/resume_comparison.py
6.95 kB
| """Resume the persistent experiment after workspace interruption. | |
| Run: python3 scripts/resume_comparison.py | |
| Never restarts a live driver or automatically retries a recorded training error. | |
| """ | |
| import ctypes.util,fcntl,json,os,pathlib,shutil,subprocess,sys,time | |
| import launch_comparison as pipeline | |
| from workspace_storage import activate | |
| ROOT=pipeline.ROOT;LOCAL=pipeline.LOCAL | |
| def restore_system_libraries(): | |
| if ctypes.util.find_library('GL') and ctypes.util.find_library('OSMesa'):return | |
| packages=sorted((LOCAL/'system-packages').glob('*.deb')) | |
| if not packages:raise RuntimeError('Persistent renderer packages missing; restore system libraries first') | |
| pipeline.run('restore-system-libraries',['dpkg','-i',*packages]) | |
| def restore_python_runtime(): | |
| if pipeline.PYTHON.is_file():return | |
| archive=LOCAL/'runtime-env.tar.gz' | |
| if not archive.exists(): | |
| pipeline.run('download-python-runtime',[LOCAL/'hf-backup-venv/bin/python',ROOT/'scripts/download_runtime_archive.py']) | |
| archive=pathlib.Path('/tmp/robottt-runtime-download/environment/runtime-env.tar.gz') | |
| pipeline.run('restore-python-runtime',[sys.executable,ROOT/'scripts/archive_runtime.py','restore','--archive',archive,'--output','/tmp/robottt-runtime/venv']) | |
| for alias,target in [(LOCAL/'venv-persistent',pathlib.Path('/tmp/robottt-runtime/venv')),(LOCAL/'venv',LOCAL/'venv-persistent')]: | |
| if not alias.exists() and not alias.is_symlink():alias.symlink_to(target,target_is_directory=True) | |
| def workers(): | |
| for name,python,args in [('monitor',sys.executable,['scripts/monitor_comparison.py','--watch']),('hf-backup',str(LOCAL/'hf-backup-venv/bin/python'),['scripts/backup_huggingface.py','--watch'])]: | |
| pidfile=LOCAL/(name+'-worker.pid') | |
| if pidfile.exists(): | |
| command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' | |
| if command.exists() and args[0].encode() in command.read_bytes():continue | |
| with (LOCAL/'audit'/(name+'-worker.log')).open('a') as log: | |
| process=subprocess.Popen([python,*args],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) | |
| pidfile.write_text(str(process.pid)) | |
| if (LOCAL/'completion-watch.json').exists(): | |
| pidfile=LOCAL/'completion-watch-worker.pid' | |
| command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' if pidfile.exists() else None | |
| if command is None or not command.exists() or b'watch_comparison.py' not in command.read_bytes(): | |
| with (LOCAL/'audit/completion-watch-worker.log').open('a') as log: | |
| process=subprocess.Popen([sys.executable,'scripts/watch_comparison.py','--watch'],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) | |
| pidfile.write_text(str(process.pid)) | |
| def main(): | |
| os.chdir(ROOT) | |
| activated=activate() | |
| lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| support=LOCAL/'runtime-support.tar.gz' | |
| if support.exists():pipeline.run('restore-runtime-support',[sys.executable,ROOT/'scripts/archive_runtime.py','restore-support','--archive',support]) | |
| restore_python_runtime() | |
| if activated:pipeline.run('restore-remote-progress',[pipeline.PYTHON,ROOT/'scripts/restore_remote_progress.py']) | |
| workers();restore_system_libraries() | |
| gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name','--format=csv,noheader'],text=True).strip() | |
| uuid,name=[value.strip() for value in gpu.split(',')] | |
| if name!='NVIDIA GeForce RTX 3090':raise RuntimeError('Replacement GPU model requires validation') | |
| pipeline.write(LOCAL/'audit'/f'resume-hardware-{time.time_ns()}.json',dict(gpu=gpu,updated_unix_time=time.time())) | |
| runs=json.loads((LOCAL/'common-artifacts.json').read_text())['run_configs'] | |
| for kind,config in runs.items(): | |
| rp=pathlib.Path(config);r=json.loads(rp.read_text());out=pathlib.Path(r['paths']['output']) | |
| if (out/'completed.json').exists(): | |
| pipeline.run('verify-completed-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume']) | |
| elif (out/'checkpoints/latest.pt').exists(): | |
| pipeline.run('recover-saved-'+kind,[pipeline.PYTHON,ROOT/'scripts/recover_checkpoint.py','--run-config',rp]) | |
| pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume']) | |
| else: | |
| out.mkdir(parents=True,exist_ok=True) | |
| with (out/'run.lock').open('a') as run_lock: | |
| fcntl.flock(run_lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| if list(out.glob('failure*.json')):raise RuntimeError('Recorded failure needs diagnosis') | |
| journal=out/'ledger/events.jsonl' | |
| if journal.exists() and journal.stat().st_size: | |
| rows=[json.loads(line) for line in journal.read_text().splitlines()] | |
| if any(row.get('kind')=='optimizer_update' for row in rows):raise RuntimeError('Missing checkpoint after optimizer attempts; restore it from HF') | |
| archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True) | |
| shutil.copyfile(rp,archive/'config.json');shutil.move(journal,archive/'events.jsonl') | |
| r['gpu_uuid']=uuid;r['limits']['model_load']=1024;pipeline.write(rp,r) | |
| pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp]) | |
| flow=LOCAL/'evaluations'/('flow-'+kind);summary=flow/'summary.json' | |
| if not summary.exists(): | |
| if flow.exists(): | |
| archive=LOCAL/'evaluations/interruption-archives';archive.mkdir(parents=True,exist_ok=True) | |
| shutil.move(flow,archive/(flow.name+'-'+str(time.time_ns()))) | |
| pipeline.run('flow-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',flow]) | |
| elif json.loads(summary.read_text())['run_config_sha256']!=pipeline.digest(rp):raise ValueError('Completed flow evaluation identity differs') | |
| for kind,config in runs.items(): | |
| output=LOCAL/'evaluations'/('libero-'+kind);summary=output/'summary.json' | |
| if summary.exists() and json.loads(summary.read_text()).get('complete'):continue | |
| pipeline.run('libero-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',config,'--plan',LOCAL/'libero-eval-plan.json','--output',output]) | |
| pipeline.run('summarize',[pipeline.PYTHON,ROOT/'scripts/summarize_comparison.py']) | |
| pipeline.stage('completed','completed') | |
| if __name__=='__main__': | |
| try:main() | |
| except BlockingIOError:raise SystemExit('An experiment driver is already active; no changes made.') | |
| except BaseException as error: | |
| pipeline.stage('recovery_failed','failed',error=type(error).__name__+': '+str(error));raise | |