File size: 11,052 Bytes
0f5d220 5ceffa9 a02e3b4 0f5d220 add1d5f 0f5d220 add1d5f 0f5d220 e2453d6 0f5d220 add1d5f 0f5d220 add1d5f a61a6da 0f5d220 c62a767 0f5d220 add1d5f a61a6da 0f5d220 add1d5f 0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | """Durable sequential preparation, smoke, paired training and evaluation driver.
Uses only new local assets. Stops on errors; never retries an optimizer attempt.
"""
import argparse,json,os,pathlib,shutil,subprocess,sys,time,traceback,signal
ROOT=pathlib.Path(__file__).resolve().parents[1]
LOCAL=ROOT/'.release_local';PYTHON=LOCAL/'venv/bin/python';OPENPI=ROOT.parent/'openpi'
CACHE=pathlib.Path('/tmp/robottt-comparison/prefix-cache')
STATUS=LOCAL/'comparison-status.json'
ENV={**os.environ,'PYTHONDONTWRITEBYTECODE':'1','CUDA_VISIBLE_DEVICES':'0','JAX_PLATFORMS':'cpu','OMP_NUM_THREADS':'4','MKL_NUM_THREADS':'4','OPENBLAS_NUM_THREADS':'4','UV_LINK_MODE':'copy','MUJOCO_GL':'osmesa','PYOPENGL_PLATFORM':'osmesa','LP_NUM_THREADS':'4','LIBERO_CONFIG_PATH':str(LOCAL/'libero-config'),'PYTHONPATH':':'.join(map(str,[ROOT/'src',OPENPI/'src',OPENPI/'packages/openpi-client/src',OPENPI/'third_party/libero']))}
ENV.update(XDG_CACHE_HOME='/tmp/robottt-xdg-cache',MPLCONFIGDIR='/tmp/robottt-matplotlib',MESA_SHADER_CACHE_DIR='/tmp/robottt-mesa-shaders',NUMBA_CACHE_DIR='/tmp/robottt-numba-cache')
def write(path,value):
path.parent.mkdir(parents=True,exist_ok=True);tmp=path.with_name(path.name+'.tmp');tmp.write_text(json.dumps(value,indent=2));tmp.replace(path)
def stage(name,state='running',**kw):
write(STATUS,dict(stage=name,state=state,pid=os.getpid(),updated_unix_time=time.time(),**kw));print(name,state,flush=True)
def run(name,command,*,env=ENV):
log=LOCAL/'audit'/f'pipeline-{name}.log';stage(name,log=str(log))
with log.open('a') as f:
f.write('\nCOMMAND '+repr(list(map(str,command)))+'\n');f.flush()
subprocess.run(list(map(str,command)),cwd=ROOT,env=env,stdout=f,stderr=subprocess.STDOUT,check=True)
def wait_for(name,test,seconds=7200):
stage(name);deadline=time.monotonic()+seconds
while not test():
if time.monotonic()>deadline:raise TimeoutError(name)
stage(name,waiting_seconds=seconds-(deadline-time.monotonic()))
time.sleep(30)
def digest(path):
import hashlib
h=hashlib.sha256()
with pathlib.Path(path).open('rb') as f:
while b:=f.read(8*1024*1024):h.update(b)
return h.hexdigest()
def make_run(name,kind,protocol,index,updates,*,shared=None,on_demand=False):
config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
shared=protocol if shared is None else shared
paths=dict(openpi_source=OPENPI,model_checkpoint=LOCAL/'assets/pi05_base_pytorch/model.safetensors',
normalization=LOCAL/'assets/processed-libero/normalization.json',tokenizer=LOCAL/'assets/paligemma_tokenizer.model',
data_root=LOCAL/'assets/processed-libero',prefix_root=CACHE,asset_index=index,initial_extension=protocol/'initial.pt',
schedule=shared/'schedule.json',bank=shared/'bank.pt',output=LOCAL/'runs'/name,experiment_config=config)
if updates==3000:paths['runtime_profile_parity']=LOCAL/'audit/runtime-profile-parity.json'
if on_demand:
paths['prefix_root']=LOCAL/'ondemand-prefix';paths['prefix_contracts']=LOCAL/'prefix-contracts'
r=dict(paths={k:str(v) for k,v in paths.items()},hashes={k:digest(paths[k]) for k in (('model_checkpoint','asset_index','initial_extension','schedule','bank','runtime_profile_parity') if updates==3000 else ('model_checkpoint','asset_index','initial_extension','schedule','bank'))},
gpu_uuid=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip(),updates=updates,milestones=[1,2] if updates==2 else [1000,2000,3000],
checkpoint_every=1 if updates==2 else 25,runtime_profile='audited' if updates==2 else 'efficient',model_profile='pi05_base_libero_protocol',optimizer_profile='opentau_3000',
limits=dict(model_load=2,expert=updates*32,inner=updates*32*(4 if kind=='lite' else 18),backward=updates*4,optimizer_update=updates,rollout=0),original=False)
if on_demand:r['prefix_mode']='on_demand'
path=LOCAL/(name+'.json');write(path,r);return path
def prepare(name,kind,updates):
protocol=LOCAL/name
config=ROOT/'configs'/('l4_learned_t32_s8.json' if kind=='lite' else 'opentau_reference_t32_s8.json')
run(name,[PYTHON,'-m','robottt_pi05','prepare-protocol','--experiment-config',config,'--asset-index',LOCAL/'assets/processed-libero/asset-index.json','--output',protocol,'--updates',updates,'--seed','1000'])
return protocol
def main():
parser=argparse.ArgumentParser();parser.add_argument('--continue-after-smoke',action='store_true');parser.add_argument('--prefix-ready',action='store_true');parser.add_argument('--on-demand',action='store_true');args=parser.parse_args()
if args.on_demand and not args.continue_after_smoke:raise ValueError('On-demand mode requires completed integration smokes')
if args.prefix_ready and not args.continue_after_smoke:raise ValueError('Prepared prefix reuse requires verified smokes')
os.chdir(ROOT)
# A nonblocking process lock prevents accidental duplicate launchers.
import fcntl
lock=(LOCAL/'comparison-driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
prefix_cmd=[PYTHON,ROOT/'scripts/cache_prefixes.py','--openpi-source',OPENPI,'--checkpoint',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--data',LOCAL/'assets/processed-libero','--output',CACHE,'--batch-size',os.environ.get('ROBOTTT_PREFIX_BATCH_SIZE','32')]
if not args.continue_after_smoke:
wait_for('waiting_official_base_download',lambda:(LOCAL/'audit/pi05-base-download.json').exists())
pidfile=LOCAL/'counterpart-download.pid'
if pidfile.exists():
pid=int(pidfile.read_text());cmd=pathlib.Path(f'/proc/{pid}/cmdline')
if cmd.exists() and b'scripts/download_file.py' in cmd.read_bytes():os.kill(pid,signal.SIGCONT)
wait_for('waiting_dev_preprocessing',lambda:len(json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text()).get('dev_ids',[]))==10)
run('convert_official_base',[PYTHON,ROOT/'scripts/convert_base.py','--openpi-source',OPENPI,'--checkpoint-dir',LOCAL/'assets/pi05_base','--output',LOCAL/'assets/pi05_base_pytorch'],env={**ENV,'CUDA_VISIBLE_DEVICES':''})
official=LOCAL/'assets/pi05_base'
if not official.is_symlink():
relocated=pathlib.Path('/tmp/robottt-official-pi05-base')
stage('relocate_original_base_cache')
shutil.move(str(official),str(relocated));official.symlink_to(relocated,target_is_directory=True)
run('prefix_smoke',prefix_cmd+['--max-prefixes','32'])
index=LOCAL/'smoke-asset-index.json';shutil.copyfile(CACHE/'asset-index.json',index)
smoke_lite=prepare('protocol-smoke-lite','lite',2)
schedule=[dict(index=i,episode=0,start=0,source_rows=list(range(32)),split='train',key=f'smoke_{i}') for i in range(2)]
write(smoke_lite/'schedule.json',schedule)
for kind in ('lite','reference'):
protocol=smoke_lite if kind=='lite' else prepare('protocol-smoke-reference','reference',2)
rp=make_run('smoke-'+kind,kind,protocol,index,2,shared=smoke_lite)
run('smoke-'+kind+'-boundary',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--stop-after','1'])
run('smoke-'+kind+'-resume',[PYTHON,'-m','robottt_pi05','train','--run-config',rp,'--resume'])
else:
for kind in ('lite','reference'):
done=json.loads((LOCAL/'runs'/('smoke-'+kind)/'completed.json').read_text())
if done['updates']!=2 or digest(done['checkpoint']['path'])!=done['checkpoint']['sha256']:raise ValueError('Smoke boundary proof differs')
if list((LOCAL/'runs'/('smoke-'+kind)).glob('failure*.json')):raise ValueError('Smoke failure needs audit')
stage('validated_completed_smokes')
comparison_index=CACHE/'asset-index.json'
if args.on_demand:
comparison_index=LOCAL/'ondemand-prefix/asset-index.json'
manifest=json.loads((LOCAL/'assets/processed-libero/asset-index.json').read_text())
manifest['prefixes']={};manifest['protocol']['model_sha256']=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors')
manifest['protocol']['prefix_mode']='bounded_ram_on_demand_fixed_sequence_batch32_v1'
write(comparison_index,manifest)
run('validate-on-demand-runtime',[PYTHON,ROOT/'scripts/validate_cached_runtime.py','--on-demand'])
elif args.prefix_ready:
cached=json.loads((CACHE/'asset-index.json').read_text());done=json.loads((CACHE/'completed.json').read_text())
expected=sum(min(32,len(rec['manifest']['rows'])) if int(ep) in cached['dev_ids'] else len(rec['manifest']['rows']) for ep,rec in cached['episodes'].items())
if done['model_sha256']!=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors') or done['prefixes']!=expected or len(cached['prefixes'])!=expected:raise ValueError('Incomplete or different prefix cache')
if any(not (CACHE/rec['path']).is_file() for rec in cached['prefixes'].values()):raise FileNotFoundError('Cached prefix missing')
stage('verified_completed_prefix_cache')
else:run('prefix_full',prefix_cmd)
external=pathlib.Path('/tmp/robottt-comparison/tensorauto-pi05-base.safetensors')
if external.exists():
run('audit-external-base',[PYTHON,ROOT/'scripts/audit_external_base.py','--official',LOCAL/'assets/pi05_base_pytorch/model.safetensors','--external',external,'--output',LOCAL/'audit/external-base-tensor-audit.json'])
lite=prepare('protocol-comparison-lite','lite',3000)
reference=prepare('protocol-comparison-reference','reference',3000)
runs={kind:make_run('comparison-'+kind,kind,protocol,comparison_index,3000,shared=lite,on_demand=args.on_demand) for kind,protocol in [('lite',lite),('reference',reference)]}
write(LOCAL/'common-artifacts.json',dict(shared_schedule_sha256=digest(lite/'schedule.json'),shared_bank_sha256=digest(lite/'bank.pt'),shared_model_sha256=digest(LOCAL/'assets/pi05_base_pytorch/model.safetensors'),run_configs={k:str(v) for k,v in runs.items()}))
for kind,rp in runs.items():
run('train-'+kind,[PYTHON,'-m','robottt_pi05','train','--run-config',rp])
run('flow-'+kind,[PYTHON,ROOT/'scripts/evaluate_flow.py','--run-config',rp,'--output',LOCAL/'evaluations'/('flow-'+kind)])
for kind,rp in runs.items():
run('libero-'+kind+'-smoke',[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-smoke-'+kind),'--max-episodes','1'])
run('libero-'+kind,[PYTHON,ROOT/'scripts/evaluate_libero.py','--run-config',rp,'--plan',LOCAL/'libero-eval-plan.json','--output',LOCAL/'evaluations'/('libero-'+kind)])
run('summarize',[PYTHON,ROOT/'scripts/summarize_comparison.py'])
stage('completed','completed')
if __name__=='__main__':
try:main()
except BaseException as e:
stage('failed','failed',error=type(e).__name__+': '+str(e),traceback=traceback.format_exc());raise
|