File size: 6,949 Bytes
07a447a
 
 
 
 
 
 
a02e3b4
07a447a
 
 
 
 
 
 
 
 
 
 
6028162
 
 
21d44eb
 
 
6028162
6de2e12
 
6028162
 
07a447a
 
 
 
 
 
 
5ceffa9
07a447a
e2d338b
 
 
 
 
 
 
07a447a
 
 
 
a02e3b4
07a447a
2cdcdbd
 
a02e3b4
 
 
07a447a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Resume the persistent experiment after workspace interruption.

Run: python3 scripts/resume_comparison.py
Never restarts a live driver or automatically retries a recorded training error.
"""
import ctypes.util,fcntl,json,os,pathlib,shutil,subprocess,sys,time
import launch_comparison as pipeline
from workspace_storage import activate

ROOT=pipeline.ROOT;LOCAL=pipeline.LOCAL


def restore_system_libraries():
    if ctypes.util.find_library('GL') and ctypes.util.find_library('OSMesa'):return
    packages=sorted((LOCAL/'system-packages').glob('*.deb'))
    if not packages:raise RuntimeError('Persistent renderer packages missing; restore system libraries first')
    pipeline.run('restore-system-libraries',['dpkg','-i',*packages])


def restore_python_runtime():
    if pipeline.PYTHON.is_file():return
    archive=LOCAL/'runtime-env.tar.gz'
    if not archive.exists():
        pipeline.run('download-python-runtime',[LOCAL/'hf-backup-venv/bin/python',ROOT/'scripts/download_runtime_archive.py'])
        archive=pathlib.Path('/tmp/robottt-runtime-download/environment/runtime-env.tar.gz')
    pipeline.run('restore-python-runtime',[sys.executable,ROOT/'scripts/archive_runtime.py','restore','--archive',archive,'--output','/tmp/robottt-runtime/venv'])
    for alias,target in [(LOCAL/'venv-persistent',pathlib.Path('/tmp/robottt-runtime/venv')),(LOCAL/'venv',LOCAL/'venv-persistent')]:
        if not alias.exists() and not alias.is_symlink():alias.symlink_to(target,target_is_directory=True)


def workers():
    for name,python,args in [('monitor',sys.executable,['scripts/monitor_comparison.py','--watch']),('hf-backup',str(LOCAL/'hf-backup-venv/bin/python'),['scripts/backup_huggingface.py','--watch'])]:
        pidfile=LOCAL/(name+'-worker.pid')
        if pidfile.exists():
            command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline'
            if command.exists() and args[0].encode() in command.read_bytes():continue
        with (LOCAL/'audit'/(name+'-worker.log')).open('a') as log:
            process=subprocess.Popen([python,*args],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
        pidfile.write_text(str(process.pid))
    if (LOCAL/'completion-watch.json').exists():
        pidfile=LOCAL/'completion-watch-worker.pid'
        command=pathlib.Path('/proc')/pidfile.read_text().strip()/'cmdline' if pidfile.exists() else None
        if command is None or not command.exists() or b'watch_comparison.py' not in command.read_bytes():
            with (LOCAL/'audit/completion-watch-worker.log').open('a') as log:
                process=subprocess.Popen([sys.executable,'scripts/watch_comparison.py','--watch'],cwd=ROOT,env=pipeline.ENV,stdin=subprocess.DEVNULL,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
            pidfile.write_text(str(process.pid))


def main():
    os.chdir(ROOT)
    activated=activate()
    lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    support=LOCAL/'runtime-support.tar.gz'
    if support.exists():pipeline.run('restore-runtime-support',[sys.executable,ROOT/'scripts/archive_runtime.py','restore-support','--archive',support])
    restore_python_runtime()
    if activated:pipeline.run('restore-remote-progress',[pipeline.PYTHON,ROOT/'scripts/restore_remote_progress.py'])
    workers();restore_system_libraries()
    gpu=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid,name','--format=csv,noheader'],text=True).strip()
    uuid,name=[value.strip() for value in gpu.split(',')]
    if name!='NVIDIA GeForce RTX 3090':raise RuntimeError('Replacement GPU model requires validation')
    pipeline.write(LOCAL/'audit'/f'resume-hardware-{time.time_ns()}.json',dict(gpu=gpu,updated_unix_time=time.time()))
    runs=json.loads((LOCAL/'common-artifacts.json').read_text())['run_configs']
    for kind,config in runs.items():
        rp=pathlib.Path(config);r=json.loads(rp.read_text());out=pathlib.Path(r['paths']['output'])
        if (out/'completed.json').exists():
            pipeline.run('verify-completed-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
        elif (out/'checkpoints/latest.pt').exists():
            pipeline.run('recover-saved-'+kind,[pipeline.PYTHON,ROOT/'scripts/recover_checkpoint.py','--run-config',rp])
            pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
        else:
            out.mkdir(parents=True,exist_ok=True)
            with (out/'run.lock').open('a') as run_lock:
                fcntl.flock(run_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
                if list(out.glob('failure*.json')):raise RuntimeError('Recorded failure needs diagnosis')
                journal=out/'ledger/events.jsonl'
                if journal.exists() and journal.stat().st_size:
                    rows=[json.loads(line) for line in journal.read_text().splitlines()]
                    if any(row.get('kind')=='optimizer_update' for row in rows):raise RuntimeError('Missing checkpoint after optimizer attempts; restore it from HF')
                    archive=out/'interruption-archives'/str(time.time_ns());archive.mkdir(parents=True)
                    shutil.copyfile(rp,archive/'config.json');shutil.move(journal,archive/'events.jsonl')
                r['gpu_uuid']=uuid;r['limits']['model_load']=1024;pipeline.write(rp,r)
            pipeline.run('train-'+kind,[pipeline.PYTHON,'-m','robottt_pi05','train','--run-config',rp])
        flow=LOCAL/'evaluations'/('flow-'+kind);summary=flow/'summary.json'
        if not summary.exists():
            if flow.exists():
                archive=LOCAL/'evaluations/interruption-archives';archive.mkdir(parents=True,exist_ok=True)
                shutil.move(flow,archive/(flow.name+'-'+str(time.time_ns())))
            pipeline.run('flow-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',flow])
        elif json.loads(summary.read_text())['run_config_sha256']!=pipeline.digest(rp):raise ValueError('Completed flow evaluation identity differs')
    for kind,config in runs.items():
        output=LOCAL/'evaluations'/('libero-'+kind);summary=output/'summary.json'
        if summary.exists() and json.loads(summary.read_text()).get('complete'):continue
        pipeline.run('libero-'+kind,[pipeline.PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',config,'--plan',LOCAL/'libero-eval-plan.json','--output',output])
    pipeline.run('summarize',[pipeline.PYTHON,ROOT/'scripts/summarize_comparison.py'])
    pipeline.stage('completed','completed')


if __name__=='__main__':
    try:main()
    except BlockingIOError:raise SystemExit('An experiment driver is already active; no changes made.')
    except BaseException as error:
        pipeline.stage('recovery_failed','failed',error=type(error).__name__+': '+str(error));raise