Download code/src/robottt_pi05/runner.py from jisue/robottt-pi05-base-comparison: direct link, hf CLI and curl.
- Browser
- Download file 17.9 kB
-
https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/runner.py
- Command line
-
hf download hf://jisue/robottt-pi05-base-comparison/code/src/robottt_pi05/runner.py
-
curl -L -o runner.py https://huggingface.co/jisue/robottt-pi05-base-comparison/resolve/main/code/src/robottt_pi05/runner.py
17.9 kB
| """Explicit offline train/evaluate entrypoints. Importing never starts a run.""" | |
| from pathlib import Path | |
| import fcntl,gc,json,time,os,subprocess | |
| import torch | |
| from .config import ExperimentConfig,ADAMW_SETTINGS | |
| from .modeling import create_adapter,configure_trainables | |
| from .openpi_backend import load_model,Original | |
| from .assets import read_json,FrozenPrefixStore,ProcessedSequenceSource,SavedSchedule,prefix_signature,prefix_unchanged | |
| from .eval_io import atomic_json,atomic_tensor,sha,parameter_hash,state_hash | |
| from .checkpoint import load_initial,load_weights,save_boundary,restore_boundary,resume_step,restore_rng,cpu_tree | |
| from .smoke_optimizer import build_optimizer | |
| from .optimization import settings as optimizer_settings,learning_rate | |
| from .training_io import RunLedger | |
| from .recovery import DurableEvents | |
| from .engine import train_sequence,check_gradients,SafetyGuard,generate_chunk | |
| from .layer_state import packed,unpacked | |
| def _setup(run_path): | |
| run_path=Path(run_path).resolve();r=read_json(run_path);base=run_path.parent | |
| def path(key):return (base/r['paths'][key]).resolve() | |
| output=path('output');output.mkdir(parents=True,exist_ok=True) | |
| config=ExperimentConfig.load(path('experiment_config')) | |
| manifest=read_json(path('asset_index')) | |
| if r.get('model_profile')=='pi05_base_libero_protocol': | |
| expected=r['hashes']['model_checkpoint'] | |
| if manifest.get('protocol',{}).get('model_sha256',expected)!=expected:raise ValueError('Prefix model origin differs from loaded base') | |
| if any(rec.get('provenance',{}).get('model_sha256')!=expected for rec in manifest['prefixes'].values()): | |
| raise ValueError('Prefix entries belong to a different backbone') | |
| provenance={k:sha(path(k)) for k in ['experiment_config','asset_index']} | |
| provenance['run_config']=sha(run_path) | |
| if r.get('model_profile')=='pi05_base_libero_protocol': | |
| from .profile_common import runtime_signature | |
| provenance['implementation_sha256']=runtime_signature() | |
| if r.get('runtime_profile')=='efficient': | |
| proof=read_json(path('runtime_profile_parity')) | |
| if proof.get('implementation_sha256')!=provenance['implementation_sha256']:raise ValueError('Runtime parity proof belongs to different code') | |
| if proof['model_sha256']!=r['hashes']['model_checkpoint'] or proof['data_source_index_sha256']!=sha(path('data_root')/'asset-index.json'): | |
| raise ValueError('Runtime parity proof belongs to different assets') | |
| if proof['config_sha256'][config.method]!=sha(path('experiment_config')):raise ValueError('Runtime parity config differs') | |
| for key,expected in r['hashes'].items(): | |
| if sha(path(key))!=expected:raise ValueError('External asset hash mismatch: '+key) | |
| lock=(output/'run.lock').open('a') | |
| try: | |
| fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) | |
| ledger=RunLedger(output/'ledger',r['limits']) | |
| if r.get('runtime_profile') not in (None,'audited','minimal','efficient'):raise ValueError('Unknown runtime profile') | |
| ledger.defer_readonly_fsync=r.get('runtime_profile') in ('minimal','efficient') | |
| except BaseException: | |
| lock.close();raise | |
| return r,path,output,lock,config,manifest,provenance,ledger | |
| def _gpu_preflight(expected_uuid): | |
| info=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip() | |
| apps=subprocess.check_output(['nvidia-smi','-i','0','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader'],text=True) | |
| if info!=expected_uuid or info in apps or os.environ.get('CUDA_VISIBLE_DEVICES') not in ('0',expected_uuid): | |
| raise RuntimeError('Configured GPU0 must be idle and exclusively selected before loading') | |
| def _completed(out,provenance): | |
| p=out/'completed.json' | |
| if not p.exists():return None | |
| z=read_json(p) | |
| if z['provenance']!=provenance:raise ValueError('Completed run provenance mismatch') | |
| for record in z.get('outputs',[]): | |
| if sha(record['path'])!=record['sha256']:raise ValueError('Completed prediction changed') | |
| if 'checkpoint' in z and sha(z['checkpoint']['path'])!=z['checkpoint']['sha256']: | |
| raise ValueError('Completed checkpoint changed') | |
| return z | |
| def train(run_path,*,resume=False,stop_after=None): | |
| if stop_after is not None and (type(stop_after) is not int or stop_after<1):raise ValueError('Positive stop-after count required') | |
| setup=_setup(run_path) | |
| try:return _train(setup,resume=resume,stop_after=stop_after) | |
| finally:setup[3].close() | |
| def _train(setup,*,resume=False,stop_after=None): | |
| r,path,out,lock,config,m,provenance,ledger=setup | |
| done=_completed(out,provenance) | |
| if done is not None:return done | |
| if ledger.events.exists() and ledger.events.stat().st_size and not resume:raise RuntimeError('Existing run: inspect before explicit --resume') | |
| if list(out.glob('failure*.json')):raise RuntimeError('Failure preserved; audit before recovery') | |
| step=resume_step(out/'checkpoints',ledger) | |
| schedule=SavedSchedule(path('schedule'),path('bank'),schedule_sha256=r['hashes']['schedule'],bank_sha256=r['hashes']['bank'],train_ids=m['train_ids'],sequence_length=config.sequence_length,stride=config.observation_stride) | |
| interval=r.get('checkpoint_every',1) | |
| if type(interval) is not int or interval<1:raise ValueError('Positive checkpoint interval required') | |
| total=r['updates'] | |
| if type(total)is not int or not 0<total<=len(schedule):raise ValueError('Invalid fixed schedule length') | |
| if step>total:raise ValueError('Checkpoint beyond configured training length') | |
| if step==total: | |
| checkpoint=out/'checkpoints/latest.pt' | |
| saved=torch.load(checkpoint,map_location='cpu',weights_only=True) | |
| if saved['config']!=config.to_dict() or saved['provenance']!=provenance:raise ValueError('Final boundary provenance mismatch') | |
| durable=DurableEvents(ledger) | |
| for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous)) | |
| result={'status':'completed','updates':total,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed,'checkpoint':{'path':str(checkpoint),'sha256':sha(checkpoint)}} | |
| atomic_json(out/'completed.json',result);return result | |
| _gpu_preflight(r['gpu_uuid']) | |
| ledger.reserve('model_load','train') | |
| model=load_model(source=path('openpi_source'),checkpoint=path('model_checkpoint'),**({'profile':r['model_profile']} if 'model_profile' in r else {}));ledger.complete('model_load','train') | |
| guard=SafetyGuard(lr_bounds=(1e-6,1.)) if config.method=='opentau_reference' else SafetyGuard();scope=['train'] | |
| def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0]) | |
| adapter,slow=create_adapter(model,config,observer=observer,guard=guard,**({'batched_segments':True} if r.get('runtime_profile')=='efficient' else {})) | |
| adapter.minimal_diagnostics=r.get('runtime_profile') in ('minimal','efficient') | |
| adapter.use_batched_segments=r.get('runtime_profile')=='efficient' | |
| adapter.layer_observer=None if adapter.minimal_diagnostics else lambda key,phase:guard.learning_rates({key:adapter.ttt[key]}) | |
| names,params=configure_trainables(slow,config) | |
| optimizer_profile=r.get('optimizer_profile','legacy') | |
| settings,clip_norm=optimizer_settings(optimizer_profile) | |
| optimizer=build_optimizer(params,settings) | |
| if step or (out/'checkpoints/latest.pt').exists(): | |
| restored=restore_boundary(out/'checkpoints/latest.pt',slow=slow,optimizer=optimizer,config=config,provenance=provenance) | |
| if restored['step']!=step:raise ValueError('Resume boundary mismatch') | |
| durable=DurableEvents(ledger) | |
| for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous)) | |
| else: | |
| initial=load_initial(path('initial_extension'),slow,expected_sha256=r['hashes']['initial_extension']);restore_rng(initial['rng']) | |
| save_boundary(out/'checkpoints',step=0,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=ledger.completed) | |
| atomic_tensor(out/'checkpoints/step_000000.pt',torch.load(out/'checkpoints/latest.pt',map_location='cpu',weights_only=True)) | |
| source=ProcessedSequenceSource(m['episodes'],root=path('data_root'),allowed_episodes=m['train_ids'],stride=config.observation_stride) | |
| prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=m['train_ids']) | |
| if r.get('prefix_mode')=='on_demand': | |
| from .ondemand_prefix import OnDemandPrefixStore | |
| prefixes=OnDemandPrefixStore(model,m,data_root=path('data_root'),contract_root=path('prefix_contracts'),model_sha256=r['hashes']['model_checkpoint'],allowed_episodes=m['train_ids']) | |
| frozen=[(p,p._version) for p in model.parameters()] | |
| atomic_json(out/'model_audit.json',{'base_frozen':all(not p.requires_grad for p in model.parameters()), | |
| 'base_parameters':sum(p.numel() for p in model.parameters()),'extension_trainable_parameters':sum(p.numel() for p in params), | |
| 'extension_trainable_tensors':len(params),'fast_state_scalars_per_trajectory':sum(w.numel() for layer in adapter.ttt.values() for w in layer.w0),'timesteps_per_expert_batch':8 if adapter.use_batched_segments else 1,'parameter_names':names,'model_sha256':r['hashes'].get('model_checkpoint')},replace=True) | |
| timings=[] | |
| try: | |
| for index in range(step,total): | |
| for group in optimizer.param_groups:group['lr']=learning_rate(index,optimizer_profile) | |
| sc,eps,tm=schedule[index];scope[0]=str(index+1);begin=time.perf_counter() | |
| prepared=prefixes.sequence(sc['episode'],sc['source_rows']);signature=prefix_signature(prepared) | |
| if hasattr(prefixes,'prefetch') and index+1<total:prefixes.prefetch(schedule.rows[index+1]['episode']) | |
| actions=source.targets(sc['episode'],sc['start'],config.sequence_length).cuda();optimizer.zero_grad(set_to_none=True) | |
| guard.learning_rates(adapter.ttt) | |
| def backward(loss): | |
| ledger.reserve('backward',scope[0]);loss.backward();guard();ledger.complete('backward',scope[0]) | |
| result=train_sequence(adapter,config,prepared,actions,eps.cuda(),tm.cuda(),backward=backward) | |
| gradients=check_gradients(slow) | |
| if not prefix_unchanged(signature) or any(p.grad is not None or p._version!=v for p,v in frozen):raise RuntimeError('Frozen model/prefix mutated') | |
| norm=float(torch.nn.utils.clip_grad_norm_(params,clip_norm));guard.learning_rates(adapter.ttt) | |
| ledger.reserve('optimizer_update',scope[0]);optimizer.step();guard();guard.learning_rates(adapter.ttt) | |
| if any(not torch.isfinite(p).all() for p in params):raise FloatingPointError('Nonfinite slow weights') | |
| committed=dict(ledger.completed);committed['optimizer_update']+=1 | |
| boundary=(index+1)%interval==0 or index+1==total or index+1 in r.get('milestones',[]) or (stop_after is not None and index+1-step>=stop_after) | |
| saved=None | |
| if boundary: | |
| _,saved=save_boundary(out/'checkpoints',step=index+1,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=committed) | |
| if index+1 in r.get('milestones',[256,512,1024]):atomic_tensor(out/f'checkpoints/step_{index+1:06d}.pt',saved) | |
| ledger.complete('optimizer_update',scope[0]) | |
| ledger.event('training_step',step=index+1,schedule_index=index,loss=result['loss'],learning_rate=optimizer.param_groups[0]['lr'],gradients=gradients,pre_clip_norm=norm,inner_lr={k:float(v.inner_lr.detach()) for k,v in adapter.ttt.items()}) | |
| elapsed=time.perf_counter()-begin;timings.append(elapsed) | |
| ledger.profile(scope=scope[0],values={'seconds':elapsed,'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30}) | |
| median=sum(timings[-50:])/len(timings[-50:]) | |
| atomic_json(out/'status.json',{'stage':'training','completed_updates':index+1,'total_updates':total, | |
| 'flow_matching_loss':result['loss'],'seconds_per_update_recent':median,'remaining_seconds_estimate':median*(total-index-1), | |
| 'updated_unix_time':time.time(),'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30},replace=True) | |
| adapter.history.clear();adapter.call_contracts.clear();del result,prepared,signature,actions,saved;gc.collect() | |
| if stop_after is not None and index+1-step>=stop_after and index+1<total: | |
| return {'status':'boundary_saved','updates':index+1,'remaining':total-index-1} | |
| except BaseException as error: | |
| atomic_json(out/f'failure_{time.time_ns()}.json',{'type':type(error).__name__,'message':str(error),'automatic_retry':False});raise | |
| checkpoint=out/'checkpoints/latest.pt' | |
| result={'status':'completed','updates':total,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed,'checkpoint':{'path':str(checkpoint),'sha256':sha(checkpoint)}} | |
| atomic_json(out/'completed.json',result) | |
| atomic_json(out/'status.json',{'stage':'training_completed','completed_updates':total,'total_updates':total,'updated_unix_time':time.time()},replace=True) | |
| return result | |
| def evaluate(run_path): | |
| setup=_setup(run_path) | |
| try:return _evaluate(setup) | |
| finally:setup[3].close() | |
| def _evaluate(setup): | |
| r,path,out,lock,config,m,provenance,ledger=setup | |
| done=_completed(out,provenance) | |
| if done is not None:return done | |
| plan=read_json(path('evaluation_plan'));bank=torch.load(path('bank'),map_location='cpu',weights_only=True)['evaluation_noise'] | |
| allowed=set(m['train_ids'])|set(m['dev_ids']) | |
| if len({e['key'] for e in plan})!=len(plan):raise ValueError('Duplicate evaluation key') | |
| for e in plan: | |
| if e['episode'] not in allowed or e['mode'] not in ('persistent','reset') or e['noise_id'] not in (0,1):raise PermissionError('Evaluation allowlist/mode') | |
| if e['timesteps']!=list(range(16)):raise ValueError('Release CLI supports start0 T16 evaluation') | |
| _gpu_preflight(r['gpu_uuid']) | |
| ledger.reserve('model_load','eval');model=load_model(source=path('openpi_source'),checkpoint=path('model_checkpoint'),**({'profile':r['model_profile']} if 'model_profile' in r else {}));ledger.complete('model_load','eval') | |
| guard=SafetyGuard();scope=['eval'] | |
| def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0]) | |
| if r.get('original',False):adapter=Original(model,observer,guard);slow=None | |
| else: | |
| adapter,slow=create_adapter(model,config,observer=observer,guard=guard,inference=True) | |
| load_weights(path('extension_checkpoint'),slow,config,expected_sha256=r['hashes']['extension_checkpoint']) | |
| frozen_hash=None if slow is None else parameter_hash(slow.state_dict()) | |
| prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=allowed) | |
| outputs=[];durable=DurableEvents(ledger) | |
| for case in plan: | |
| folder=out/case['key'];folder.mkdir(exist_ok=True);final=folder/'sequence.pt' | |
| identity={'case':case,'provenance':provenance,'checkpoint':r['hashes'].get('extension_checkpoint','Original')} | |
| if final.exists(): | |
| z=torch.load(final,map_location='cpu',weights_only=True) | |
| if z['identity']!=identity:raise ValueError('Completed identity mismatch') | |
| for rec in z['chunks']: | |
| if sha(rec['path'])!=rec['sha256']:raise ValueError('Completed chunk changed') | |
| outputs.append({'path':str(final),'sha256':sha(final)});continue | |
| state=adapter.initial_state();predictions=[];chunks=[] | |
| for t in case['timesteps']: | |
| scope[0]=case['key']+f'_t{t}';p=folder/f'chunk_{t:02d}.pt' | |
| if case['mode']=='reset':state=adapter.initial_state() | |
| incoming={k:state_hash(s) for k,s in packed(state).items()} | |
| if p.exists(): | |
| z=torch.load(p,map_location='cpu',weights_only=True) | |
| if z['identity']!=identity or z['t']!=t or z['incoming']!=incoming:raise ValueError('Chunk recovery mismatch') | |
| if scope[0] in ledger.outputs and sha(p)!=ledger.outputs[scope[0]]['sha256']:raise ValueError('Stored chunk hash mismatch') | |
| durable.artifact('rollout',scope[0]) | |
| state=unpacked(z['candidate'],device='cuda');predictions.append(z['prediction']);chunks.append({'path':str(p),'sha256':sha(p)});continue | |
| prepared=prefixes.get(case['episode'],10*t);signature=prefix_signature([prepared]) | |
| ledger.reserve('rollout',scope[0]);prediction,candidate,solver=generate_chunk(adapter,prepared,bank[case['noise_id'],:,t].cuda(),state) | |
| if not prefix_unchanged(signature):raise RuntimeError('Prefix mutated') | |
| if slow is not None and parameter_hash(slow.state_dict())!=frozen_hash:raise RuntimeError('Slow inference values changed') | |
| z={'identity':identity,'t':t,'prediction':prediction.cpu(),'candidate':packed(candidate),'incoming':incoming,'solver':solver} | |
| atomic_tensor(p,z);ledger.complete('rollout',scope[0]);ledger.output_complete(key=scope[0],path=p) | |
| state=candidate;predictions.append(z['prediction']);chunks.append({'path':str(p),'sha256':sha(p)}) | |
| if hasattr(adapter,'history'):adapter.history.clear();adapter.call_contracts.clear() | |
| atomic_tensor(final,{'identity':identity,'prediction':torch.stack(predictions,1),'chunks':chunks,'target_free':True}) | |
| outputs.append({'path':str(final),'sha256':sha(final)}) | |
| result={'status':'completed','outputs':outputs,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed} | |
| atomic_json(out/'completed.json',result);return result | |