"""Explicit offline train/evaluate entrypoints. Importing never starts a run.""" from pathlib import Path import fcntl,gc,json,time,os,subprocess import torch from .config import ExperimentConfig,ADAMW_SETTINGS from .modeling import create_adapter,configure_trainables from .openpi_backend import load_model,Original from .assets import read_json,FrozenPrefixStore,ProcessedSequenceSource,SavedSchedule,prefix_signature,prefix_unchanged from .eval_io import atomic_json,atomic_tensor,sha,parameter_hash,state_hash from .checkpoint import load_initial,load_weights,save_boundary,restore_boundary,resume_step,restore_rng,cpu_tree from .smoke_optimizer import build_optimizer from .optimization import settings as optimizer_settings,learning_rate from .training_io import RunLedger from .recovery import DurableEvents from .engine import train_sequence,check_gradients,SafetyGuard,generate_chunk from .layer_state import packed,unpacked def _setup(run_path): run_path=Path(run_path).resolve();r=read_json(run_path);base=run_path.parent def path(key):return (base/r['paths'][key]).resolve() output=path('output');output.mkdir(parents=True,exist_ok=True) config=ExperimentConfig.load(path('experiment_config')) manifest=read_json(path('asset_index')) if r.get('model_profile')=='pi05_base_libero_protocol': expected=r['hashes']['model_checkpoint'] if manifest.get('protocol',{}).get('model_sha256',expected)!=expected:raise ValueError('Prefix model origin differs from loaded base') if any(rec.get('provenance',{}).get('model_sha256')!=expected for rec in manifest['prefixes'].values()): raise ValueError('Prefix entries belong to a different backbone') provenance={k:sha(path(k)) for k in ['experiment_config','asset_index']} provenance['run_config']=sha(run_path) if r.get('model_profile')=='pi05_base_libero_protocol': from .profile_common import runtime_signature provenance['implementation_sha256']=runtime_signature() if r.get('runtime_profile')=='efficient': proof=read_json(path('runtime_profile_parity')) if proof.get('implementation_sha256')!=provenance['implementation_sha256']:raise ValueError('Runtime parity proof belongs to different code') if proof['model_sha256']!=r['hashes']['model_checkpoint'] or proof['data_source_index_sha256']!=sha(path('data_root')/'asset-index.json'): raise ValueError('Runtime parity proof belongs to different assets') if proof['config_sha256'][config.method]!=sha(path('experiment_config')):raise ValueError('Runtime parity config differs') for key,expected in r['hashes'].items(): if sha(path(key))!=expected:raise ValueError('External asset hash mismatch: '+key) lock=(output/'run.lock').open('a') try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) ledger=RunLedger(output/'ledger',r['limits']) if r.get('runtime_profile') not in (None,'audited','minimal','efficient'):raise ValueError('Unknown runtime profile') ledger.defer_readonly_fsync=r.get('runtime_profile') in ('minimal','efficient') except BaseException: lock.close();raise return r,path,output,lock,config,manifest,provenance,ledger def _gpu_preflight(expected_uuid): info=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip() apps=subprocess.check_output(['nvidia-smi','-i','0','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader'],text=True) if info!=expected_uuid or info in apps or os.environ.get('CUDA_VISIBLE_DEVICES') not in ('0',expected_uuid): raise RuntimeError('Configured GPU0 must be idle and exclusively selected before loading') def _completed(out,provenance): p=out/'completed.json' if not p.exists():return None z=read_json(p) if z['provenance']!=provenance:raise ValueError('Completed run provenance mismatch') for record in z.get('outputs',[]): if sha(record['path'])!=record['sha256']:raise ValueError('Completed prediction changed') if 'checkpoint' in z and sha(z['checkpoint']['path'])!=z['checkpoint']['sha256']: raise ValueError('Completed checkpoint changed') return z def train(run_path,*,resume=False,stop_after=None): if stop_after is not None and (type(stop_after) is not int or stop_after<1):raise ValueError('Positive stop-after count required') setup=_setup(run_path) try:return _train(setup,resume=resume,stop_after=stop_after) finally:setup[3].close() def _train(setup,*,resume=False,stop_after=None): r,path,out,lock,config,m,provenance,ledger=setup done=_completed(out,provenance) if done is not None:return done if ledger.events.exists() and ledger.events.stat().st_size and not resume:raise RuntimeError('Existing run: inspect before explicit --resume') if list(out.glob('failure*.json')):raise RuntimeError('Failure preserved; audit before recovery') step=resume_step(out/'checkpoints',ledger) schedule=SavedSchedule(path('schedule'),path('bank'),schedule_sha256=r['hashes']['schedule'],bank_sha256=r['hashes']['bank'],train_ids=m['train_ids'],sequence_length=config.sequence_length,stride=config.observation_stride) interval=r.get('checkpoint_every',1) if type(interval) is not int or interval<1:raise ValueError('Positive checkpoint interval required') total=r['updates'] if type(total)is not int or not 0total:raise ValueError('Checkpoint beyond configured training length') if step==total: checkpoint=out/'checkpoints/latest.pt' saved=torch.load(checkpoint,map_location='cpu',weights_only=True) if saved['config']!=config.to_dict() or saved['provenance']!=provenance:raise ValueError('Final boundary provenance mismatch') durable=DurableEvents(ledger) for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous)) result={'status':'completed','updates':total,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed,'checkpoint':{'path':str(checkpoint),'sha256':sha(checkpoint)}} atomic_json(out/'completed.json',result);return result _gpu_preflight(r['gpu_uuid']) ledger.reserve('model_load','train') model=load_model(source=path('openpi_source'),checkpoint=path('model_checkpoint'),**({'profile':r['model_profile']} if 'model_profile' in r else {}));ledger.complete('model_load','train') guard=SafetyGuard(lr_bounds=(1e-6,1.)) if config.method=='opentau_reference' else SafetyGuard();scope=['train'] def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0]) adapter,slow=create_adapter(model,config,observer=observer,guard=guard,**({'batched_segments':True} if r.get('runtime_profile')=='efficient' else {})) adapter.minimal_diagnostics=r.get('runtime_profile') in ('minimal','efficient') adapter.use_batched_segments=r.get('runtime_profile')=='efficient' adapter.layer_observer=None if adapter.minimal_diagnostics else lambda key,phase:guard.learning_rates({key:adapter.ttt[key]}) names,params=configure_trainables(slow,config) optimizer_profile=r.get('optimizer_profile','legacy') settings,clip_norm=optimizer_settings(optimizer_profile) optimizer=build_optimizer(params,settings) if step or (out/'checkpoints/latest.pt').exists(): restored=restore_boundary(out/'checkpoints/latest.pt',slow=slow,optimizer=optimizer,config=config,provenance=provenance) if restored['step']!=step:raise ValueError('Resume boundary mismatch') durable=DurableEvents(ledger) for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous)) else: initial=load_initial(path('initial_extension'),slow,expected_sha256=r['hashes']['initial_extension']);restore_rng(initial['rng']) save_boundary(out/'checkpoints',step=0,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=ledger.completed) atomic_tensor(out/'checkpoints/step_000000.pt',torch.load(out/'checkpoints/latest.pt',map_location='cpu',weights_only=True)) source=ProcessedSequenceSource(m['episodes'],root=path('data_root'),allowed_episodes=m['train_ids'],stride=config.observation_stride) prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=m['train_ids']) if r.get('prefix_mode')=='on_demand': from .ondemand_prefix import OnDemandPrefixStore prefixes=OnDemandPrefixStore(model,m,data_root=path('data_root'),contract_root=path('prefix_contracts'),model_sha256=r['hashes']['model_checkpoint'],allowed_episodes=m['train_ids']) frozen=[(p,p._version) for p in model.parameters()] atomic_json(out/'model_audit.json',{'base_frozen':all(not p.requires_grad for p in model.parameters()), 'base_parameters':sum(p.numel() for p in model.parameters()),'extension_trainable_parameters':sum(p.numel() for p in params), 'extension_trainable_tensors':len(params),'fast_state_scalars_per_trajectory':sum(w.numel() for layer in adapter.ttt.values() for w in layer.w0),'timesteps_per_expert_batch':8 if adapter.use_batched_segments else 1,'parameter_names':names,'model_sha256':r['hashes'].get('model_checkpoint')},replace=True) timings=[] try: for index in range(step,total): for group in optimizer.param_groups:group['lr']=learning_rate(index,optimizer_profile) sc,eps,tm=schedule[index];scope[0]=str(index+1);begin=time.perf_counter() prepared=prefixes.sequence(sc['episode'],sc['source_rows']);signature=prefix_signature(prepared) if hasattr(prefixes,'prefetch') and index+1=stop_after) saved=None if boundary: _,saved=save_boundary(out/'checkpoints',step=index+1,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=committed) if index+1 in r.get('milestones',[256,512,1024]):atomic_tensor(out/f'checkpoints/step_{index+1:06d}.pt',saved) ledger.complete('optimizer_update',scope[0]) ledger.event('training_step',step=index+1,schedule_index=index,loss=result['loss'],learning_rate=optimizer.param_groups[0]['lr'],gradients=gradients,pre_clip_norm=norm,inner_lr={k:float(v.inner_lr.detach()) for k,v in adapter.ttt.items()}) elapsed=time.perf_counter()-begin;timings.append(elapsed) ledger.profile(scope=scope[0],values={'seconds':elapsed,'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30}) median=sum(timings[-50:])/len(timings[-50:]) atomic_json(out/'status.json',{'stage':'training','completed_updates':index+1,'total_updates':total, 'flow_matching_loss':result['loss'],'seconds_per_update_recent':median,'remaining_seconds_estimate':median*(total-index-1), 'updated_unix_time':time.time(),'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30},replace=True) adapter.history.clear();adapter.call_contracts.clear();del result,prepared,signature,actions,saved;gc.collect() if stop_after is not None and index+1-step>=stop_after and index+1