robottt-pi05-base-comparison / code /scripts /launch_comparison.py
jisue's picture
Experiment backup 2026-11-10T10:55:21.529008+00:00
a02e3b4 verified
Raw History Blame Contribute Delete
11.1 kB
"""Durable sequential preparation, smoke, paired training and evaluation driver.
Uses only new local assets. Stops on errors; never retries an optimizer attempt.
"""
import argparse,json,os,pathlib,shutil,subprocess,sys,time,traceback,signal
ROOT=pathlib.Path(__file__).resolve().parents[1]
LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi'
CACHE=pathlib.Path('/tmp/robottt-comparison/prefix-cache')
STATUS=LOCAL/'comparison-status.json'
ENV={**os.environ,'PYTHONDONTWRITEBYTECODE':'1','CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','UV_LINK_MODE':'copy','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))}
ENV.update(XDG_CACHE_HOME='/tmp/robottt-xdg-cache',MPLCONFIGDIR='/tmp/robottt-matplotlib',MESA_SHADER_CACHE_DIR='/tmp/robottt-mesa-shaders',NUMBA_CACHE_DIR='/tmp/robottt-numba-cache')
def write(path,value):
path.parent.mkdir(parents=True,exist_ok=True);tmp=path.with_name(path.name+'.tmp');tmp.write_text(json.dumps(value,indent=2));tmp.replace(path)
def stage(name,state='running',**kw):
write(STATUS,dict(stage=name,state=state,pid=os.getpid(),updated_unix_time=time.time(),**kw));print(name,state,flush=True)
def run(name,command,*,env=ENV):
log=LOCAL/'audit'/f'pipeline-{name}.log';stage(name,log=str(log))
with log.open('a') as f:
f.write('\nCOMMAND '+repr(list(map(str,command)))+'\n');f.flush()
subprocess.run(list(map(str,command)),cwd=ROOT,env=env,stdout=f,stderr=subprocess.STDOUT,check=True)
def wait_for(name,test,seconds=7200):
stage(name);deadline=time.monotonic()+seconds
while not test():
if time.monotonic()>deadline:raise TimeoutError(name)
stage(name,waiting_seconds=seconds-(deadline-time.monotonic()))
time.sleep(30)
def digest(path):
import hashlib
h=hashlib.sha256()
with pathlib.Path(path).open('rb') as f:
while b:=f.read(8*1024*1024):h.update(b)
return h.hexdigest()
def make_run(name,kind,protocol,index,updates,*,shared=None,on_demand=False):
config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
shared=protocol if shared is None else shared
paths=dict(openpi_source=OPENPI,model_checkpoint=LOCAL/'assets/pi05_base_pytorch/model.safetensors',
normalization=LOCAL/'assets/processed-libero/normalization.json',tokenizer=LOCAL/'assets/paligemma_tokenizer.model',
data_root=LOCAL/'assets/processed-libero',prefix_root=CACHE,asset_index=index,initial_extension=protocol/'initial.pt',
schedule=shared/'schedule.json',bank=shared/'bank.pt',output=LOCAL/'runs'/name,experiment_config=config)
if updates==3000:paths['runtime_profile_parity']=LOCAL/'audit/runtime-profile-parity.json'
if on_demand:
paths['prefix_root']=LOCAL/'ondemand-prefix';paths['prefix_contracts']=LOCAL/'prefix-contracts'
r=dict(paths={k:str(v) for k,v in paths.items()},hashes={k:digest(paths[k]) for k in (('model_checkpoint','asset_index','initial_extension','schedule','bank','runtime_profile_parity') if updates==3000 else ('model_checkpoint','asset_index','initial_extension','schedule','bank'))},
gpu_uuid=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip(),updates=updates,milestones=[1,2] if updates==2 else [1000,2000,3000],
checkpoint_every=1 if updates==2 else 25,runtime_profile='audited' if updates==2 else 'efficient',model_profile='pi05_base_libero_protocol',optimizer_profile='opentau_3000',
limits=dict(model_load=2,expert=updates*32,inner=updates*32*(4 if kind=='lite' else 18),backward=updates*4,optimizer_update=updates,rollout=0),original=False)
if on_demand:r['prefix_mode']='on_demand'
path=LOCAL/(name+'.json');write(path,r);return path
def prepare(name,kind,updates):
protocol=LOCAL/name
config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
run(name,[PYTHON,'-m','robottt_pi05','prepare-protocol','--experiment-config',config,'--asset-index',LOCAL/'assets/processed-libero/asset-index.json','--output',protocol,'--updates',updates,'--seed','1000'])
return protocol
def main():
parser=argparse.ArgumentParser();parser.add_argument('--continue-after-smoke',action='store_true');parser.add_argument('--prefix-ready',action='store_true');parser.add_argument('--on-demand',action='store_true');args=parser.parse_args()
if args.on_demand and not args.continue_after_smoke:raise ValueError('On-demand mode requires completed integration smokes')
if args.prefix_ready and not args.continue_after_smoke:raise ValueError('Prepared prefix reuse requires verified smokes')
os.chdir(ROOT)
# A nonblocking process lock prevents accidental duplicate launchers.
import fcntl
lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
prefix_cmd=[PYTHON,ROOT/'scripts/cache_prefixes.py','--openpi-source',OPENPI,'--checkpoint',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--data',LOCAL/'assets/processed-libero','--output',CACHE,'--batch-size',os.environ.get('ROBOTTT_PREFIX_BATCH_SIZE','32')]
if not args.continue_after_smoke:
wait_for('waiting_official_base_download',lambda:(LOCAL/'audit/pi05-base-download.json').exists())
pidfile=LOCAL/'counterpart-download.pid'
if pidfile.exists():
pid=int(pidfile.read_text());cmd=pathlib.Path(f'/proc/{pid}/cmdline')
if cmd.exists() and b'scripts/download_file.py' in cmd.read_bytes():os.kill(pid,signal.SIGCONT)
wait_for('waiting_dev_preprocessing',lambda:len(json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text()).get('dev_ids',[]))==10)
run('convert_official_base',[PYTHON,ROOT/'scripts/convert_base.py','--openpi-source',OPENPI,'--checkpoint-dir',LOCAL/'assets/pi05_base','--output',LOCAL/'assets/pi05_base_pytorch'],env={**ENV,'CUDA_VISIBLE_DEVICES':''})
official=LOCAL/'assets/pi05_base'
if not official.is_symlink():
relocated=pathlib.Path('/tmp/robottt-official-pi05-base')
stage('relocate_original_base_cache')
shutil.move(str(official),str(relocated));official.symlink_to(relocated,target_is_directory=True)
run('prefix_smoke',prefix_cmd+['--max-prefixes','32'])
index=LOCAL/'smoke-asset-index.json';shutil.copyfile(CACHE/'asset-index.json',index)
smoke_lite=prepare('protocol-smoke-lite','lite',2)
schedule=[dict(index=i,episode=0,start=0,source_rows=list(range(32)),split='train',key=f'smoke_{i}') for i in range(2)]
write(smoke_lite/'schedule.json',schedule)
for kind in ('lite','reference'):
protocol=smoke_lite if kind=='lite' else prepare('protocol-smoke-reference','reference',2)
rp=make_run('smoke-'+kind,kind,protocol,index,2,shared=smoke_lite)
run('smoke-'+kind+'-boundary',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--stop-after','1'])
run('smoke-'+kind+'-resume',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
else:
for kind in ('lite','reference'):
done=json.loads((LOCAL/'runs'/('smoke-'+kind)/'completed.json').read_text())
if done['updates']!=2 or digest(done['checkpoint']['path'])!=done['checkpoint']['sha256']:raise ValueError('Smoke boundary proof differs')
if list((LOCAL/'runs'/('smoke-'+kind)).glob('failure*.json')):raise ValueError('Smoke failure needs audit')
stage('validated_completed_smokes')
comparison_index=CACHE/'asset-index.json'
if args.on_demand:
comparison_index=LOCAL/'ondemand-prefix/asset-index.json'
manifest=json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text())
manifest['prefixes']={};manifest['protocol']['model_sha256']=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors')
manifest['protocol']['prefix_mode']='bounded_ram_on_demand_fixed_sequence_batch32_v1'
write(comparison_index,manifest)
run('validate-on-demand-runtime',[PYTHON,ROOT/'scripts/validate_cached_runtime.py','--on-demand'])
elif args.prefix_ready:
cached=json.loads((CACHE/'asset-index.json').read_text());done=json.loads((CACHE/'completed.json').read_text())
expected=sum(min(32,len(rec['manifest']['rows'])) if int(ep) in cached['dev_ids'] else len(rec['manifest']['rows']) for ep,rec in cached['episodes'].items())
if done['model_sha256']!=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors') or done['prefixes']!=expected or len(cached['prefixes'])!=expected:raise ValueError('Incomplete or different prefix cache')
if any(not (CACHE/rec['path']).is_file() for rec in cached['prefixes'].values()):raise FileNotFoundError('Cached prefix missing')
stage('verified_completed_prefix_cache')
else:run('prefix_full',prefix_cmd)
external=pathlib.Path('/tmp/robottt-comparison/tensorauto-pi05-base.safetensors')
if external.exists():
run('audit-external-base',[PYTHON,ROOT/'scripts/audit_external_base.py','--official',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--external',external,'--output',LOCAL/'audit/external-base-tensor-audit.json'])
lite=prepare('protocol-comparison-lite','lite',3000)
reference=prepare('protocol-comparison-reference','reference',3000)
runs={kind:make_run('comparison-'+kind,kind,protocol,comparison_index,3000,shared=lite,on_demand=args.on_demand) for kind,protocol in [('lite',lite),('reference',reference)]}
write(LOCAL/'common-artifacts.json',dict(shared_schedule_sha256=digest(lite/'schedule.json'),shared_bank_sha256=digest(lite/'bank.pt'),shared_model_sha256=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors'),run_configs={k:str(v) for k,v in runs.items()}))
for kind,rp in runs.items():
run('train-'+kind,[PYTHON,'-m','robottt_pi05','train','--run-config',rp])
run('flow-'+kind,[PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',LOCAL/'evaluations'/('flow-'+kind)])
for kind,rp in runs.items():
run('libero-'+kind+'-smoke',[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-smoke-'+kind),'--max-episodes','1'])
run('libero-'+kind,[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-'+kind)])
run('summarize',[PYTHON,ROOT/'scripts/summarize_comparison.py'])
stage('completed','completed')
if __name__=='__main__':
try:main()
except BaseException as e:
stage('failed','failed',error=type(e).__name__+': '+str(e),traceback=traceback.format_exc());raise