"""Durable sequential preparation, smoke, paired training and evaluation driver. Uses only new local assets. Stops on errors; never retries an optimizer attempt. """ import argparse,json,os,pathlib,shutil,subprocess,sys,time,traceback,signal ROOT=pathlib.Path(__file__).resolve().parents[1] LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi' CACHE=pathlib.Path('/tmp/robottt-comparison/prefix-cache') STATUS=LOCAL/'comparison-status.json' ENV={**os.environ,'PYTHONDONTWRITEBYTECODE':'1','CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','UV_LINK_MODE':'copy','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))} ENV.update(XDG_CACHE_HOME='/tmp/robottt-xdg-cache',MPLCONFIGDIR='/tmp/robottt-matplotlib',MESA_SHADER_CACHE_DIR='/tmp/robottt-mesa-shaders',NUMBA_CACHE_DIR='/tmp/robottt-numba-cache') def write(path,value): path.parent.mkdir(parents=True,exist_ok=True);tmp=path.with_name(path.name+'.tmp');tmp.write_text(json.dumps(value,indent=2));tmp.replace(path) def stage(name,state='running',**kw): write(STATUS,dict(stage=name,state=state,pid=os.getpid(),updated_unix_time=time.time(),**kw));print(name,state,flush=True) def run(name,command,*,env=ENV): log=LOCAL/'audit'/f'pipeline-{name}.log';stage(name,log=str(log)) with log.open('a') as f: f.write('\nCOMMAND '+repr(list(map(str,command)))+'\n');f.flush() subprocess.run(list(map(str,command)),cwd=ROOT,env=env,stdout=f,stderr=subprocess.STDOUT,check=True) def wait_for(name,test,seconds=7200): stage(name);deadline=time.monotonic()+seconds while not test(): if time.monotonic()>deadline:raise TimeoutError(name) stage(name,waiting_seconds=seconds-(deadline-time.monotonic())) time.sleep(30) def digest(path): import hashlib h=hashlib.sha256() with pathlib.Path(path).open('rb') as f: while b:=f.read(8*1024*1024):h.update(b) return h.hexdigest() def make_run(name,kind,protocol,index,updates,*,shared=None,on_demand=False): config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json') shared=protocol if shared is None else shared paths=dict(openpi_source=OPENPI,model_checkpoint=LOCAL/'assets/pi05_base_pytorch/model.safetensors', normalization=LOCAL/'assets/processed-libero/normalization.json',tokenizer=LOCAL/'assets/paligemma_tokenizer.model', data_root=LOCAL/'assets/processed-libero',prefix_root=CACHE,asset_index=index,initial_extension=protocol/'initial.pt', schedule=shared/'schedule.json',bank=shared/'bank.pt',output=LOCAL/'runs'/name,experiment_config=config) if updates==3000:paths['runtime_profile_parity']=LOCAL/'audit/runtime-profile-parity.json' if on_demand: paths['prefix_root']=LOCAL/'ondemand-prefix';paths['prefix_contracts']=LOCAL/'prefix-contracts' r=dict(paths={k:str(v) for k,v in paths.items()},hashes={k:digest(paths[k]) for k in (('model_checkpoint','asset_index','initial_extension','schedule','bank','runtime_profile_parity') if updates==3000 else ('model_checkpoint','asset_index','initial_extension','schedule','bank'))}, gpu_uuid=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip(),updates=updates,milestones=[1,2] if updates==2 else [1000,2000,3000], checkpoint_every=1 if updates==2 else 25,runtime_profile='audited' if updates==2 else 'efficient',model_profile='pi05_base_libero_protocol',optimizer_profile='opentau_3000', limits=dict(model_load=2,expert=updates*32,inner=updates*32*(4 if kind=='lite' else 18),backward=updates*4,optimizer_update=updates,rollout=0),original=False) if on_demand:r['prefix_mode']='on_demand' path=LOCAL/(name+'.json');write(path,r);return path def prepare(name,kind,updates): protocol=LOCAL/name config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json') run(name,[PYTHON,'-m','robottt_pi05','prepare-protocol','--experiment-config',config,'--asset-index',LOCAL/'assets/processed-libero/asset-index.json','--output',protocol,'--updates',updates,'--seed','1000']) return protocol def main(): parser=argparse.ArgumentParser();parser.add_argument('--continue-after-smoke',action='store_true');parser.add_argument('--prefix-ready',action='store_true');parser.add_argument('--on-demand',action='store_true');args=parser.parse_args() if args.on_demand and not args.continue_after_smoke:raise ValueError('On-demand mode requires completed integration smokes') if args.prefix_ready and not args.continue_after_smoke:raise ValueError('Prepared prefix reuse requires verified smokes') os.chdir(ROOT) # A nonblocking process lock prevents accidental duplicate launchers. import fcntl lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) prefix_cmd=[PYTHON,ROOT/'scripts/cache_prefixes.py','--openpi-source',OPENPI,'--checkpoint',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--data',LOCAL/'assets/processed-libero','--output',CACHE,'--batch-size',os.environ.get('ROBOTTT_PREFIX_BATCH_SIZE','32')] if not args.continue_after_smoke: wait_for('waiting_official_base_download',lambda:(LOCAL/'audit/pi05-base-download.json').exists()) pidfile=LOCAL/'counterpart-download.pid' if pidfile.exists(): pid=int(pidfile.read_text());cmd=pathlib.Path(f'/proc/{pid}/cmdline') if cmd.exists() and b'scripts/download_file.py' in cmd.read_bytes():os.kill(pid,signal.SIGCONT) wait_for('waiting_dev_preprocessing',lambda:len(json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text()).get('dev_ids',[]))==10) run('convert_official_base',[PYTHON,ROOT/'scripts/convert_base.py','--openpi-source',OPENPI,'--checkpoint-dir',LOCAL/'assets/pi05_base','--output',LOCAL/'assets/pi05_base_pytorch'],env={**ENV,'CUDA_VISIBLE_DEVICES':''}) official=LOCAL/'assets/pi05_base' if not official.is_symlink(): relocated=pathlib.Path('/tmp/robottt-official-pi05-base') stage('relocate_original_base_cache') shutil.move(str(official),str(relocated));official.symlink_to(relocated,target_is_directory=True) run('prefix_smoke',prefix_cmd+['--max-prefixes','32']) index=LOCAL/'smoke-asset-index.json';shutil.copyfile(CACHE/'asset-index.json',index) smoke_lite=prepare('protocol-smoke-lite','lite',2) schedule=[dict(index=i,episode=0,start=0,source_rows=list(range(32)),split='train',key=f'smoke_{i}') for i in range(2)] write(smoke_lite/'schedule.json',schedule) for kind in ('lite','reference'): protocol=smoke_lite if kind=='lite' else prepare('protocol-smoke-reference','reference',2) rp=make_run('smoke-'+kind,kind,protocol,index,2,shared=smoke_lite) run('smoke-'+kind+'-boundary',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--stop-after','1']) run('smoke-'+kind+'-resume',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume']) else: for kind in ('lite','reference'): done=json.loads((LOCAL/'runs'/('smoke-'+kind)/'completed.json').read_text()) if done['updates']!=2 or digest(done['checkpoint']['path'])!=done['checkpoint']['sha256']:raise ValueError('Smoke boundary proof differs') if list((LOCAL/'runs'/('smoke-'+kind)).glob('failure*.json')):raise ValueError('Smoke failure needs audit') stage('validated_completed_smokes') comparison_index=CACHE/'asset-index.json' if args.on_demand: comparison_index=LOCAL/'ondemand-prefix/asset-index.json' manifest=json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text()) manifest['prefixes']={};manifest['protocol']['model_sha256']=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors') manifest['protocol']['prefix_mode']='bounded_ram_on_demand_fixed_sequence_batch32_v1' write(comparison_index,manifest) run('validate-on-demand-runtime',[PYTHON,ROOT/'scripts/validate_cached_runtime.py','--on-demand']) elif args.prefix_ready: cached=json.loads((CACHE/'asset-index.json').read_text());done=json.loads((CACHE/'completed.json').read_text()) expected=sum(min(32,len(rec['manifest']['rows'])) if int(ep) in cached['dev_ids'] else len(rec['manifest']['rows']) for ep,rec in cached['episodes'].items()) if done['model_sha256']!=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors') or done['prefixes']!=expected or len(cached['prefixes'])!=expected:raise ValueError('Incomplete or different prefix cache') if any(not (CACHE/rec['path']).is_file() for rec in cached['prefixes'].values()):raise FileNotFoundError('Cached prefix missing') stage('verified_completed_prefix_cache') else:run('prefix_full',prefix_cmd) external=pathlib.Path('/tmp/robottt-comparison/tensorauto-pi05-base.safetensors') if external.exists(): run('audit-external-base',[PYTHON,ROOT/'scripts/audit_external_base.py','--official',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--external',external,'--output',LOCAL/'audit/external-base-tensor-audit.json']) lite=prepare('protocol-comparison-lite','lite',3000) reference=prepare('protocol-comparison-reference','reference',3000) runs={kind:make_run('comparison-'+kind,kind,protocol,comparison_index,3000,shared=lite,on_demand=args.on_demand) for kind,protocol in [('lite',lite),('reference',reference)]} write(LOCAL/'common-artifacts.json',dict(shared_schedule_sha256=digest(lite/'schedule.json'),shared_bank_sha256=digest(lite/'bank.pt'),shared_model_sha256=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors'),run_configs={k:str(v) for k,v in runs.items()})) for kind,rp in runs.items(): run('train-'+kind,[PYTHON,'-m','robottt_pi05','train','--run-config',rp]) run('flow-'+kind,[PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',LOCAL/'evaluations'/('flow-'+kind)]) for kind,rp in runs.items(): run('libero-'+kind+'-smoke',[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-smoke-'+kind),'--max-episodes','1']) run('libero-'+kind,[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-'+kind)]) run('summarize',[PYTHON,ROOT/'scripts/summarize_comparison.py']) stage('completed','completed') if __name__=='__main__': try:main() except BaseException as e: stage('failed','failed',error=type(e).__name__+': '+str(e),traceback=traceback.format_exc());raise