File size: 11,052 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
5ceffa9
a02e3b4
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
add1d5f
0f5d220
 
 
 
 
 
 
add1d5f
 
0f5d220
e2453d6
0f5d220
 
add1d5f
0f5d220
 
 
 
 
 
 
 
 
add1d5f
 
a61a6da
0f5d220
 
 
 
c62a767
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
add1d5f
 
 
 
 
 
 
 
 
a61a6da
 
 
 
 
 
0f5d220
 
 
 
 
add1d5f
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
"""Durable sequential preparation, smoke, paired training and evaluation driver.

Uses only new local assets. Stops on errors; never retries an optimizer attempt.
"""
import argparse,json,os,pathlib,shutil,subprocess,sys,time,traceback,signal

ROOT=pathlib.Path(__file__).resolve().parents[1]
LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi'
CACHE=pathlib.Path('/tmp/robottt-comparison/prefix-cache')
STATUS=LOCAL/'comparison-status.json'
ENV={**os.environ,'PYTHONDONTWRITEBYTECODE':'1','CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','UV_LINK_MODE':'copy','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))}
ENV.update(XDG_CACHE_HOME='/tmp/robottt-xdg-cache',MPLCONFIGDIR='/tmp/robottt-matplotlib',MESA_SHADER_CACHE_DIR='/tmp/robottt-mesa-shaders',NUMBA_CACHE_DIR='/tmp/robottt-numba-cache')

def write(path,value):
    path.parent.mkdir(parents=True,exist_ok=True);tmp=path.with_name(path.name+'.tmp');tmp.write_text(json.dumps(value,indent=2));tmp.replace(path)

def stage(name,state='running',**kw):
    write(STATUS,dict(stage=name,state=state,pid=os.getpid(),updated_unix_time=time.time(),**kw));print(name,state,flush=True)

def run(name,command,*,env=ENV):
    log=LOCAL/'audit'/f'pipeline-{name}.log';stage(name,log=str(log))
    with log.open('a') as f:
        f.write('\nCOMMAND '+repr(list(map(str,command)))+'\n');f.flush()
        subprocess.run(list(map(str,command)),cwd=ROOT,env=env,stdout=f,stderr=subprocess.STDOUT,check=True)

def wait_for(name,test,seconds=7200):
    stage(name);deadline=time.monotonic()+seconds
    while not test():
        if time.monotonic()>deadline:raise TimeoutError(name)
        stage(name,waiting_seconds=seconds-(deadline-time.monotonic()))
        time.sleep(30)

def digest(path):
    import hashlib
    h=hashlib.sha256()
    with pathlib.Path(path).open('rb') as f:
        while b:=f.read(8*1024*1024):h.update(b)
    return h.hexdigest()

def make_run(name,kind,protocol,index,updates,*,shared=None,on_demand=False):
    config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
    shared=protocol if shared is None else shared
    paths=dict(openpi_source=OPENPI,model_checkpoint=LOCAL/'assets/pi05_base_pytorch/model.safetensors',
        normalization=LOCAL/'assets/processed-libero/normalization.json',tokenizer=LOCAL/'assets/paligemma_tokenizer.model',
        data_root=LOCAL/'assets/processed-libero',prefix_root=CACHE,asset_index=index,initial_extension=protocol/'initial.pt',
        schedule=shared/'schedule.json',bank=shared/'bank.pt',output=LOCAL/'runs'/name,experiment_config=config)
    if updates==3000:paths['runtime_profile_parity']=LOCAL/'audit/runtime-profile-parity.json'
    if on_demand:
        paths['prefix_root']=LOCAL/'ondemand-prefix';paths['prefix_contracts']=LOCAL/'prefix-contracts'
    r=dict(paths={k:str(v) for k,v in paths.items()},hashes={k:digest(paths[k]) for k in (('model_checkpoint','asset_index','initial_extension','schedule','bank','runtime_profile_parity') if updates==3000 else ('model_checkpoint','asset_index','initial_extension','schedule','bank'))},
        gpu_uuid=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip(),updates=updates,milestones=[1,2] if updates==2 else [1000,2000,3000],
        checkpoint_every=1 if updates==2 else 25,runtime_profile='audited' if updates==2 else 'efficient',model_profile='pi05_base_libero_protocol',optimizer_profile='opentau_3000',
        limits=dict(model_load=2,expert=updates*32,inner=updates*32*(4 if kind=='lite' else 18),backward=updates*4,optimizer_update=updates,rollout=0),original=False)
    if on_demand:r['prefix_mode']='on_demand'
    path=LOCAL/(name+'.json');write(path,r);return path

def prepare(name,kind,updates):
    protocol=LOCAL/name
    config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
    run(name,[PYTHON,'-m','robottt_pi05','prepare-protocol','--experiment-config',config,'--asset-index',LOCAL/'assets/processed-libero/asset-index.json','--output',protocol,'--updates',updates,'--seed','1000'])
    return protocol

def main():
    parser=argparse.ArgumentParser();parser.add_argument('--continue-after-smoke',action='store_true');parser.add_argument('--prefix-ready',action='store_true');parser.add_argument('--on-demand',action='store_true');args=parser.parse_args()
    if args.on_demand and not args.continue_after_smoke:raise ValueError('On-demand mode requires completed integration smokes')
    if args.prefix_ready and not args.continue_after_smoke:raise ValueError('Prepared prefix reuse requires verified smokes')
    os.chdir(ROOT)
    # A nonblocking process lock prevents accidental duplicate launchers.
    import fcntl
    lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
    prefix_cmd=[PYTHON,ROOT/'scripts/cache_prefixes.py','--openpi-source',OPENPI,'--checkpoint',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--data',LOCAL/'assets/processed-libero','--output',CACHE,'--batch-size',os.environ.get('ROBOTTT_PREFIX_BATCH_SIZE','32')]
    if not args.continue_after_smoke:
        wait_for('waiting_official_base_download',lambda:(LOCAL/'audit/pi05-base-download.json').exists())
        pidfile=LOCAL/'counterpart-download.pid'
        if pidfile.exists():
            pid=int(pidfile.read_text());cmd=pathlib.Path(f'/proc/{pid}/cmdline')
            if cmd.exists() and b'scripts/download_file.py' in cmd.read_bytes():os.kill(pid,signal.SIGCONT)
        wait_for('waiting_dev_preprocessing',lambda:len(json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text()).get('dev_ids',[]))==10)
        run('convert_official_base',[PYTHON,ROOT/'scripts/convert_base.py','--openpi-source',OPENPI,'--checkpoint-dir',LOCAL/'assets/pi05_base','--output',LOCAL/'assets/pi05_base_pytorch'],env={**ENV,'CUDA_VISIBLE_DEVICES':''})
        official=LOCAL/'assets/pi05_base'
        if not official.is_symlink():
            relocated=pathlib.Path('/tmp/robottt-official-pi05-base')
            stage('relocate_original_base_cache')
            shutil.move(str(official),str(relocated));official.symlink_to(relocated,target_is_directory=True)
        run('prefix_smoke',prefix_cmd+['--max-prefixes','32'])
        index=LOCAL/'smoke-asset-index.json';shutil.copyfile(CACHE/'asset-index.json',index)
        smoke_lite=prepare('protocol-smoke-lite','lite',2)
        schedule=[dict(index=i,episode=0,start=0,source_rows=list(range(32)),split='train',key=f'smoke_{i}') for i in range(2)]
        write(smoke_lite/'schedule.json',schedule)
        for kind in ('lite','reference'):
            protocol=smoke_lite if kind=='lite' else prepare('protocol-smoke-reference','reference',2)
            rp=make_run('smoke-'+kind,kind,protocol,index,2,shared=smoke_lite)
            run('smoke-'+kind+'-boundary',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--stop-after','1'])
            run('smoke-'+kind+'-resume',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
    else:
        for kind in ('lite','reference'):
            done=json.loads((LOCAL/'runs'/('smoke-'+kind)/'completed.json').read_text())
            if done['updates']!=2 or digest(done['checkpoint']['path'])!=done['checkpoint']['sha256']:raise ValueError('Smoke boundary proof differs')
            if list((LOCAL/'runs'/('smoke-'+kind)).glob('failure*.json')):raise ValueError('Smoke failure needs audit')
        stage('validated_completed_smokes')
    comparison_index=CACHE/'asset-index.json'
    if args.on_demand:
        comparison_index=LOCAL/'ondemand-prefix/asset-index.json'
        manifest=json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text())
        manifest['prefixes']={};manifest['protocol']['model_sha256']=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors')
        manifest['protocol']['prefix_mode']='bounded_ram_on_demand_fixed_sequence_batch32_v1'
        write(comparison_index,manifest)
        run('validate-on-demand-runtime',[PYTHON,ROOT/'scripts/validate_cached_runtime.py','--on-demand'])
    elif args.prefix_ready:
        cached=json.loads((CACHE/'asset-index.json').read_text());done=json.loads((CACHE/'completed.json').read_text())
        expected=sum(min(32,len(rec['manifest']['rows'])) if int(ep) in cached['dev_ids'] else len(rec['manifest']['rows']) for ep,rec in cached['episodes'].items())
        if done['model_sha256']!=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors') or done['prefixes']!=expected or len(cached['prefixes'])!=expected:raise ValueError('Incomplete or different prefix cache')
        if any(not (CACHE/rec['path']).is_file() for rec in cached['prefixes'].values()):raise FileNotFoundError('Cached prefix missing')
        stage('verified_completed_prefix_cache')
    else:run('prefix_full',prefix_cmd)
    external=pathlib.Path('/tmp/robottt-comparison/tensorauto-pi05-base.safetensors')
    if external.exists():
        run('audit-external-base',[PYTHON,ROOT/'scripts/audit_external_base.py','--official',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--external',external,'--output',LOCAL/'audit/external-base-tensor-audit.json'])
    lite=prepare('protocol-comparison-lite','lite',3000)
    reference=prepare('protocol-comparison-reference','reference',3000)
    runs={kind:make_run('comparison-'+kind,kind,protocol,comparison_index,3000,shared=lite,on_demand=args.on_demand) for kind,protocol in [('lite',lite),('reference',reference)]}
    write(LOCAL/'common-artifacts.json',dict(shared_schedule_sha256=digest(lite/'schedule.json'),shared_bank_sha256=digest(lite/'bank.pt'),shared_model_sha256=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors'),run_configs={k:str(v) for k,v in runs.items()}))
    for kind,rp in runs.items():
        run('train-'+kind,[PYTHON,'-m','robottt_pi05','train','--run-config',rp])
        run('flow-'+kind,[PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',LOCAL/'evaluations'/('flow-'+kind)])
    for kind,rp in runs.items():
        run('libero-'+kind+'-smoke',[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-smoke-'+kind),'--max-episodes','1'])
        run('libero-'+kind,[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-'+kind)])
    run('summarize',[PYTHON,ROOT/'scripts/summarize_comparison.py'])
    stage('completed','completed')

if __name__=='__main__':
    try:main()
    except BaseException as e:
        stage('failed','failed',error=type(e).__name__+': '+str(e),traceback=traceback.format_exc());raise