File size: 6,949 Bytes
07a447a a02e3b4 07a447a 6028162 21d44eb 6028162 6de2e12 6028162 07a447a 5ceffa9 07a447a e2d338b 07a447a a02e3b4 07a447a 2cdcdbd a02e3b4 07a447a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | """Resume the persistent experiment after workspace interruption.
Run: python3 scripts/resume_comparison.py
Never restarts a live driver or automatically retries a recorded training error.
"""
import ctypes.util,fcntl,json,os,pathlib,shutil,subprocess,sys,time
import launch_comparison as pipeline
from workspace_storage import activate
ROOT=pipeline.ROOT;LOCAL=pipeline.LOCAL
def restore_system_libraries():
if ctypes.util.find_library('GL') and ctypes.util.find_library('OSMesa'):return
packages=sorted((LOCAL/'system-packages').glob('*.deb'))
if not packages:raise RuntimeError('Persistent renderer packages missing; restore system libraries first')
pipeline.run('restore-system-libraries',['dpkg','-i',*packages])
def restore_python_runtime():
if pipeline.PYTHON.is_file():return
archive=LOCAL/'runtime-env.tar.gz'
if not archive.exists():
pipeline.run('download-python-runtime',[LOCAL/'hf-backup-venv/bin/python',ROOT/'scripts/download_runtime_archive.py'])
archive=pathlib.Path('/tmp/robottt-runtime-download/environment/runtime-env.tar.gz')
pipeline.run('restore-python-runtime',[sys.executable,ROOT/'scripts/archive_runtime.py','restore','--archive',archive,'--output','/tmp/robottt-runtime/venv'])
for alias,target in [(LOCAL/'venv-persistent',pathlib.Path('/tmp/robottt-runtime/venv')),(LOCAL/'venv',LOCAL/'venv-persistent')]:
if not alias.exists() and not alias.is_symlink():alias.symlink_to(target,target_is_directory=True)
def workers():
for name,python,args in [('monitor',sys.executable,['scripts/monitor_comparison.py','--watch']),('hf-backup',str(LOCAL/'hf-backup-venv/bin/python'),['scripts/backup_huggingface.py','--watch'])]:
pidfile=LOCAL/(name+'-worker.pid')
if pidfile.exists():
command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline'
if command.exists() and args[0].encode() in command.read_bytes():continue
with (LOCAL/'audit'/(name+'-worker.log')).open('a') as log:
process=subprocess.Popen([python,*args],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
pidfile.write_text(str(process.pid))
if (LOCAL/'completion-watch.json').exists():
pidfile=LOCAL/'completion-watch-worker.pid'
command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' if pidfile.exists() else None
if command is None or not command.exists() or b'watch_comparison.py' not in command.read_bytes():
with (LOCAL/'audit/completion-watch-worker.log').open('a') as log:
process=subprocess.Popen([sys.executable,'scripts/watch_comparison.py','--watch'],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
pidfile.write_text(str(process.pid))
def main():
os.chdir(ROOT)
activated=activate()
lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
support=LOCAL/'runtime-support.tar.gz'
if support.exists():pipeline.run('restore-runtime-support',[sys.executable,ROOT/'scripts/archive_runtime.py','restore-support','--archive',support])
restore_python_runtime()
if activated:pipeline.run('restore-remote-progress',[pipeline.PYTHON,ROOT/'scripts/restore_remote_progress.py'])
workers();restore_system_libraries()
gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name','--format=csv,noheader'],text=True).strip()
uuid,name=[value.strip() for value in gpu.split(',')]
if name!='NVIDIA GeForce RTX 3090':raise RuntimeError('Replacement GPU model requires validation')
pipeline.write(LOCAL/'audit'/f'resume-hardware-{time.time_ns()}.json',dict(gpu=gpu,updated_unix_time=time.time()))
runs=json.loads((LOCAL/'common-artifacts.json').read_text())['run_configs']
for kind,config in runs.items():
rp=pathlib.Path(config);r=json.loads(rp.read_text());out=pathlib.Path(r['paths']['output'])
if (out/'completed.json').exists():
pipeline.run('verify-completed-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
elif (out/'checkpoints/latest.pt').exists():
pipeline.run('recover-saved-'+kind,[pipeline.PYTHON,ROOT/'scripts/recover_checkpoint.py','--run-config',rp])
pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
else:
out.mkdir(parents=True,exist_ok=True)
with (out/'run.lock').open('a') as run_lock:
fcntl.flock(run_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if list(out.glob('failure*.json')):raise RuntimeError('Recorded failure needs diagnosis')
journal=out/'ledger/events.jsonl'
if journal.exists() and journal.stat().st_size:
rows=[json.loads(line) for line in journal.read_text().splitlines()]
if any(row.get('kind')=='optimizer_update' for row in rows):raise RuntimeError('Missing checkpoint after optimizer attempts; restore it from HF')
archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True)
shutil.copyfile(rp,archive/'config.json');shutil.move(journal,archive/'events.jsonl')
r['gpu_uuid']=uuid;r['limits']['model_load']=1024;pipeline.write(rp,r)
pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp])
flow=LOCAL/'evaluations'/('flow-'+kind);summary=flow/'summary.json'
if not summary.exists():
if flow.exists():
archive=LOCAL/'evaluations/interruption-archives';archive.mkdir(parents=True,exist_ok=True)
shutil.move(flow,archive/(flow.name+'-'+str(time.time_ns())))
pipeline.run('flow-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',flow])
elif json.loads(summary.read_text())['run_config_sha256']!=pipeline.digest(rp):raise ValueError('Completed flow evaluation identity differs')
for kind,config in runs.items():
output=LOCAL/'evaluations'/('libero-'+kind);summary=output/'summary.json'
if summary.exists() and json.loads(summary.read_text()).get('complete'):continue
pipeline.run('libero-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',config,'--plan',LOCAL/'libero-eval-plan.json','--output',output])
pipeline.run('summarize',[pipeline.PYTHON,ROOT/'scripts/summarize_comparison.py'])
pipeline.stage('completed','completed')
if __name__=='__main__':
try:main()
except BlockingIOError:raise SystemExit('An experiment driver is already active; no changes made.')
except BaseException as error:
pipeline.stage('recovery_failed','failed',error=type(error).__name__+': '+str(error));raise
|