jisue's picture
Experiment backup 2026-11-11T06:35:51.798729+00:00
fffb321 verified
Raw History Blame Contribute Delete
17.9 kB
"""Explicit offline train/evaluate entrypoints. Importing never starts a run."""
from pathlib import Path
import fcntl,gc,json,time,os,subprocess
import torch
from .config import ExperimentConfig,ADAMW_SETTINGS
from .modeling import create_adapter,configure_trainables
from .openpi_backend import load_model,Original
from .assets import read_json,FrozenPrefixStore,ProcessedSequenceSource,SavedSchedule,prefix_signature,prefix_unchanged
from .eval_io import atomic_json,atomic_tensor,sha,parameter_hash,state_hash
from .checkpoint import load_initial,load_weights,save_boundary,restore_boundary,resume_step,restore_rng,cpu_tree
from .smoke_optimizer import build_optimizer
from .optimization import settings as optimizer_settings,learning_rate
from .training_io import RunLedger
from .recovery import DurableEvents
from .engine import train_sequence,check_gradients,SafetyGuard,generate_chunk
from .layer_state import packed,unpacked
def _setup(run_path):
run_path=Path(run_path).resolve();r=read_json(run_path);base=run_path.parent
def path(key):return (base/r['paths'][key]).resolve()
output=path('output');output.mkdir(parents=True,exist_ok=True)
config=ExperimentConfig.load(path('experiment_config'))
manifest=read_json(path('asset_index'))
if r.get('model_profile')=='pi05_base_libero_protocol':
expected=r['hashes']['model_checkpoint']
if manifest.get('protocol',{}).get('model_sha256',expected)!=expected:raise ValueError('Prefix model origin differs from loaded base')
if any(rec.get('provenance',{}).get('model_sha256')!=expected for rec in manifest['prefixes'].values()):
raise ValueError('Prefix entries belong to a different backbone')
provenance={k:sha(path(k)) for k in ['experiment_config','asset_index']}
provenance['run_config']=sha(run_path)
if r.get('model_profile')=='pi05_base_libero_protocol':
from .profile_common import runtime_signature
provenance['implementation_sha256']=runtime_signature()
if r.get('runtime_profile')=='efficient':
proof=read_json(path('runtime_profile_parity'))
if proof.get('implementation_sha256')!=provenance['implementation_sha256']:raise ValueError('Runtime parity proof belongs to different code')
if proof['model_sha256']!=r['hashes']['model_checkpoint'] or proof['data_source_index_sha256']!=sha(path('data_root')/'asset-index.json'):
raise ValueError('Runtime parity proof belongs to different assets')
if proof['config_sha256'][config.method]!=sha(path('experiment_config')):raise ValueError('Runtime parity config differs')
for key,expected in r['hashes'].items():
if sha(path(key))!=expected:raise ValueError('External asset hash mismatch: '+key)
lock=(output/'run.lock').open('a')
try:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
ledger=RunLedger(output/'ledger',r['limits'])
if r.get('runtime_profile') not in (None,'audited','minimal','efficient'):raise ValueError('Unknown runtime profile')
ledger.defer_readonly_fsync=r.get('runtime_profile') in ('minimal','efficient')
except BaseException:
lock.close();raise
return r,path,output,lock,config,manifest,provenance,ledger
def _gpu_preflight(expected_uuid):
info=subprocess.check_output(['nvidia-smi','-i','0','--query-gpu=uuid','--format=csv,noheader'],text=True).strip()
apps=subprocess.check_output(['nvidia-smi','-i','0','--query-compute-apps=gpu_uuid,pid','--format=csv,noheader'],text=True)
if info!=expected_uuid or info in apps or os.environ.get('CUDA_VISIBLE_DEVICES') not in ('0',expected_uuid):
raise RuntimeError('Configured GPU0 must be idle and exclusively selected before loading')
def _completed(out,provenance):
p=out/'completed.json'
if not p.exists():return None
z=read_json(p)
if z['provenance']!=provenance:raise ValueError('Completed run provenance mismatch')
for record in z.get('outputs',[]):
if sha(record['path'])!=record['sha256']:raise ValueError('Completed prediction changed')
if 'checkpoint' in z and sha(z['checkpoint']['path'])!=z['checkpoint']['sha256']:
raise ValueError('Completed checkpoint changed')
return z
def train(run_path,*,resume=False,stop_after=None):
if stop_after is not None and (type(stop_after) is not int or stop_after<1):raise ValueError('Positive stop-after count required')
setup=_setup(run_path)
try:return _train(setup,resume=resume,stop_after=stop_after)
finally:setup[3].close()
def _train(setup,*,resume=False,stop_after=None):
r,path,out,lock,config,m,provenance,ledger=setup
done=_completed(out,provenance)
if done is not None:return done
if ledger.events.exists() and ledger.events.stat().st_size and not resume:raise RuntimeError('Existing run: inspect before explicit --resume')
if list(out.glob('failure*.json')):raise RuntimeError('Failure preserved; audit before recovery')
step=resume_step(out/'checkpoints',ledger)
schedule=SavedSchedule(path('schedule'),path('bank'),schedule_sha256=r['hashes']['schedule'],bank_sha256=r['hashes']['bank'],train_ids=m['train_ids'],sequence_length=config.sequence_length,stride=config.observation_stride)
interval=r.get('checkpoint_every',1)
if type(interval) is not int or interval<1:raise ValueError('Positive checkpoint interval required')
total=r['updates']
if type(total)is not int or not 0<total<=len(schedule):raise ValueError('Invalid fixed schedule length')
if step>total:raise ValueError('Checkpoint beyond configured training length')
if step==total:
checkpoint=out/'checkpoints/latest.pt'
saved=torch.load(checkpoint,map_location='cpu',weights_only=True)
if saved['config']!=config.to_dict() or saved['provenance']!=provenance:raise ValueError('Final boundary provenance mismatch')
durable=DurableEvents(ledger)
for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous))
result={'status':'completed','updates':total,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed,'checkpoint':{'path':str(checkpoint),'sha256':sha(checkpoint)}}
atomic_json(out/'completed.json',result);return result
_gpu_preflight(r['gpu_uuid'])
ledger.reserve('model_load','train')
model=load_model(source=path('openpi_source'),checkpoint=path('model_checkpoint'),**({'profile':r['model_profile']} if 'model_profile' in r else {}));ledger.complete('model_load','train')
guard=SafetyGuard(lr_bounds=(1e-6,1.)) if config.method=='opentau_reference' else SafetyGuard();scope=['train']
def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0])
adapter,slow=create_adapter(model,config,observer=observer,guard=guard,**({'batched_segments':True} if r.get('runtime_profile')=='efficient' else {}))
adapter.minimal_diagnostics=r.get('runtime_profile') in ('minimal','efficient')
adapter.use_batched_segments=r.get('runtime_profile')=='efficient'
adapter.layer_observer=None if adapter.minimal_diagnostics else lambda key,phase:guard.learning_rates({key:adapter.ttt[key]})
names,params=configure_trainables(slow,config)
optimizer_profile=r.get('optimizer_profile','legacy')
settings,clip_norm=optimizer_settings(optimizer_profile)
optimizer=build_optimizer(params,settings)
if step or (out/'checkpoints/latest.pt').exists():
restored=restore_boundary(out/'checkpoints/latest.pt',slow=slow,optimizer=optimizer,config=config,provenance=provenance)
if restored['step']!=step:raise ValueError('Resume boundary mismatch')
durable=DurableEvents(ledger)
for previous in range(1,step+1):durable.artifact('optimizer_update',str(previous))
else:
initial=load_initial(path('initial_extension'),slow,expected_sha256=r['hashes']['initial_extension']);restore_rng(initial['rng'])
save_boundary(out/'checkpoints',step=0,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=ledger.completed)
atomic_tensor(out/'checkpoints/step_000000.pt',torch.load(out/'checkpoints/latest.pt',map_location='cpu',weights_only=True))
source=ProcessedSequenceSource(m['episodes'],root=path('data_root'),allowed_episodes=m['train_ids'],stride=config.observation_stride)
prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=m['train_ids'])
if r.get('prefix_mode')=='on_demand':
from .ondemand_prefix import OnDemandPrefixStore
prefixes=OnDemandPrefixStore(model,m,data_root=path('data_root'),contract_root=path('prefix_contracts'),model_sha256=r['hashes']['model_checkpoint'],allowed_episodes=m['train_ids'])
frozen=[(p,p._version) for p in model.parameters()]
atomic_json(out/'model_audit.json',{'base_frozen':all(not p.requires_grad for p in model.parameters()),
'base_parameters':sum(p.numel() for p in model.parameters()),'extension_trainable_parameters':sum(p.numel() for p in params),
'extension_trainable_tensors':len(params),'fast_state_scalars_per_trajectory':sum(w.numel() for layer in adapter.ttt.values() for w in layer.w0),'timesteps_per_expert_batch':8 if adapter.use_batched_segments else 1,'parameter_names':names,'model_sha256':r['hashes'].get('model_checkpoint')},replace=True)
timings=[]
try:
for index in range(step,total):
for group in optimizer.param_groups:group['lr']=learning_rate(index,optimizer_profile)
sc,eps,tm=schedule[index];scope[0]=str(index+1);begin=time.perf_counter()
prepared=prefixes.sequence(sc['episode'],sc['source_rows']);signature=prefix_signature(prepared)
if hasattr(prefixes,'prefetch') and index+1<total:prefixes.prefetch(schedule.rows[index+1]['episode'])
actions=source.targets(sc['episode'],sc['start'],config.sequence_length).cuda();optimizer.zero_grad(set_to_none=True)
guard.learning_rates(adapter.ttt)
def backward(loss):
ledger.reserve('backward',scope[0]);loss.backward();guard();ledger.complete('backward',scope[0])
result=train_sequence(adapter,config,prepared,actions,eps.cuda(),tm.cuda(),backward=backward)
gradients=check_gradients(slow)
if not prefix_unchanged(signature) or any(p.grad is not None or p._version!=v for p,v in frozen):raise RuntimeError('Frozen model/prefix mutated')
norm=float(torch.nn.utils.clip_grad_norm_(params,clip_norm));guard.learning_rates(adapter.ttt)
ledger.reserve('optimizer_update',scope[0]);optimizer.step();guard();guard.learning_rates(adapter.ttt)
if any(not torch.isfinite(p).all() for p in params):raise FloatingPointError('Nonfinite slow weights')
committed=dict(ledger.completed);committed['optimizer_update']+=1
boundary=(index+1)%interval==0 or index+1==total or index+1 in r.get('milestones',[]) or (stop_after is not None and index+1-step>=stop_after)
saved=None
if boundary:
_,saved=save_boundary(out/'checkpoints',step=index+1,slow=slow,optimizer=optimizer,config=config,provenance=provenance,attempted=ledger.attempted,completed=committed)
if index+1 in r.get('milestones',[256,512,1024]):atomic_tensor(out/f'checkpoints/step_{index+1:06d}.pt',saved)
ledger.complete('optimizer_update',scope[0])
ledger.event('training_step',step=index+1,schedule_index=index,loss=result['loss'],learning_rate=optimizer.param_groups[0]['lr'],gradients=gradients,pre_clip_norm=norm,inner_lr={k:float(v.inner_lr.detach()) for k,v in adapter.ttt.items()})
elapsed=time.perf_counter()-begin;timings.append(elapsed)
ledger.profile(scope=scope[0],values={'seconds':elapsed,'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30})
median=sum(timings[-50:])/len(timings[-50:])
atomic_json(out/'status.json',{'stage':'training','completed_updates':index+1,'total_updates':total,
'flow_matching_loss':result['loss'],'seconds_per_update_recent':median,'remaining_seconds_estimate':median*(total-index-1),
'updated_unix_time':time.time(),'peak_cuda_gib':torch.cuda.max_memory_allocated()/2**30},replace=True)
adapter.history.clear();adapter.call_contracts.clear();del result,prepared,signature,actions,saved;gc.collect()
if stop_after is not None and index+1-step>=stop_after and index+1<total:
return {'status':'boundary_saved','updates':index+1,'remaining':total-index-1}
except BaseException as error:
atomic_json(out/f'failure_{time.time_ns()}.json',{'type':type(error).__name__,'message':str(error),'automatic_retry':False});raise
checkpoint=out/'checkpoints/latest.pt'
result={'status':'completed','updates':total,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed,'checkpoint':{'path':str(checkpoint),'sha256':sha(checkpoint)}}
atomic_json(out/'completed.json',result)
atomic_json(out/'status.json',{'stage':'training_completed','completed_updates':total,'total_updates':total,'updated_unix_time':time.time()},replace=True)
return result
def evaluate(run_path):
setup=_setup(run_path)
try:return _evaluate(setup)
finally:setup[3].close()
def _evaluate(setup):
r,path,out,lock,config,m,provenance,ledger=setup
done=_completed(out,provenance)
if done is not None:return done
plan=read_json(path('evaluation_plan'));bank=torch.load(path('bank'),map_location='cpu',weights_only=True)['evaluation_noise']
allowed=set(m['train_ids'])|set(m['dev_ids'])
if len({e['key'] for e in plan})!=len(plan):raise ValueError('Duplicate evaluation key')
for e in plan:
if e['episode'] not in allowed or e['mode'] not in ('persistent','reset') or e['noise_id'] not in (0,1):raise PermissionError('Evaluation allowlist/mode')
if e['timesteps']!=list(range(16)):raise ValueError('Release CLI supports start0 T16 evaluation')
_gpu_preflight(r['gpu_uuid'])
ledger.reserve('model_load','eval');model=load_model(source=path('openpi_source'),checkpoint=path('model_checkpoint'),**({'profile':r['model_profile']} if 'model_profile' in r else {}));ledger.complete('model_load','eval')
guard=SafetyGuard();scope=['eval']
def observer(kind,phase):getattr(ledger,'reserve' if phase=='attempt' else 'complete')(kind,scope[0])
if r.get('original',False):adapter=Original(model,observer,guard);slow=None
else:
adapter,slow=create_adapter(model,config,observer=observer,guard=guard,inference=True)
load_weights(path('extension_checkpoint'),slow,config,expected_sha256=r['hashes']['extension_checkpoint'])
frozen_hash=None if slow is None else parameter_hash(slow.state_dict())
prefixes=FrozenPrefixStore(m['prefixes'],root=path('prefix_root'),allowed_episodes=allowed)
outputs=[];durable=DurableEvents(ledger)
for case in plan:
folder=out/case['key'];folder.mkdir(exist_ok=True);final=folder/'sequence.pt'
identity={'case':case,'provenance':provenance,'checkpoint':r['hashes'].get('extension_checkpoint','Original')}
if final.exists():
z=torch.load(final,map_location='cpu',weights_only=True)
if z['identity']!=identity:raise ValueError('Completed identity mismatch')
for rec in z['chunks']:
if sha(rec['path'])!=rec['sha256']:raise ValueError('Completed chunk changed')
outputs.append({'path':str(final),'sha256':sha(final)});continue
state=adapter.initial_state();predictions=[];chunks=[]
for t in case['timesteps']:
scope[0]=case['key']+f'_t{t}';p=folder/f'chunk_{t:02d}.pt'
if case['mode']=='reset':state=adapter.initial_state()
incoming={k:state_hash(s) for k,s in packed(state).items()}
if p.exists():
z=torch.load(p,map_location='cpu',weights_only=True)
if z['identity']!=identity or z['t']!=t or z['incoming']!=incoming:raise ValueError('Chunk recovery mismatch')
if scope[0] in ledger.outputs and sha(p)!=ledger.outputs[scope[0]]['sha256']:raise ValueError('Stored chunk hash mismatch')
durable.artifact('rollout',scope[0])
state=unpacked(z['candidate'],device='cuda');predictions.append(z['prediction']);chunks.append({'path':str(p),'sha256':sha(p)});continue
prepared=prefixes.get(case['episode'],10*t);signature=prefix_signature([prepared])
ledger.reserve('rollout',scope[0]);prediction,candidate,solver=generate_chunk(adapter,prepared,bank[case['noise_id'],:,t].cuda(),state)
if not prefix_unchanged(signature):raise RuntimeError('Prefix mutated')
if slow is not None and parameter_hash(slow.state_dict())!=frozen_hash:raise RuntimeError('Slow inference values changed')
z={'identity':identity,'t':t,'prediction':prediction.cpu(),'candidate':packed(candidate),'incoming':incoming,'solver':solver}
atomic_tensor(p,z);ledger.complete('rollout',scope[0]);ledger.output_complete(key=scope[0],path=p)
state=candidate;predictions.append(z['prediction']);chunks.append({'path':str(p),'sha256':sha(p)})
if hasattr(adapter,'history'):adapter.history.clear();adapter.call_contracts.clear()
atomic_tensor(final,{'identity':identity,'prediction':torch.stack(predictions,1),'chunks':chunks,'target_free':True})
outputs.append({'path':str(final),'sha256':sha(final)})
result={'status':'completed','outputs':outputs,'provenance':provenance,'attempted':ledger.attempted,'completed':ledger.completed}
atomic_json(out/'completed.json',result);return result