"""Versioned extension checkpoints only; no base policy weights.""" from pathlib import Path import copy,hashlib,json,os,random,time import numpy as np import torch from .eval_io import sha,parameter_hash,tensor_hash,atomic_tensor def cpu_tree(v): if isinstance(v,torch.Tensor):return v.detach().cpu().clone() if isinstance(v,dict):return {k:cpu_tree(x) for k,x in v.items()} if isinstance(v,list):return [cpu_tree(x) for x in v] if isinstance(v,tuple):return tuple(cpu_tree(x) for x in v) return copy.deepcopy(v) def digest(v): h=hashlib.sha256() def visit(x): if isinstance(x,torch.Tensor):h.update(('tensor:'+tensor_hash(x)).encode()) elif isinstance(x,dict): h.update(b'dict') for k in sorted(x,key=lambda k:(type(k).__name__,str(k))):visit(k);visit(x[k]) elif isinstance(x,(list,tuple)): h.update(type(x).__name__.encode()) for y in x:visit(y) else:h.update((type(x).__name__+':'+repr(x)).encode()) visit(v);return h.hexdigest() def capture_rng(): n=np.random.get_state() return {'torch':torch.get_rng_state().clone(),'cuda':torch.cuda.get_rng_state(0).cpu().clone() if torch.cuda.is_initialized() else None, 'python':random.getstate(),'numpy':{'name':n[0],'keys':torch.from_numpy(n[1].astype(np.int64)),'position':n[2],'has_gauss':n[3],'cached':n[4]}} def restore_rng(rng): torch.set_rng_state(rng['torch']);random.setstate(rng['python']);n=rng['numpy'] np.random.set_state((n['name'],n['keys'].numpy().astype(np.uint32),n['position'],n['has_gauss'],n['cached'])) if rng['cuda'] is not None: if not torch.cuda.is_initialized():raise RuntimeError('Saved CUDA RNG requires the single already loaded GPU instance') torch.cuda.set_rng_state(rng['cuda'],0) def fsync_dir(path): fd=os.open(path,os.O_RDONLY) try:os.fsync(fd) finally:os.close(fd) SCHEMA = 'robottt_pi05.extension.v1' LEGACY = {'T16_sequence_training_v1','learned_inner_lr_v1','bptt_horizon_v1'} def strict_slow_load(slow,state): expected=slow.state_dict() if set(state)!=set(expected):raise ValueError('Missing or unexpected extension keys') for key,value in state.items(): if not isinstance(value,torch.Tensor) or value.shape!=expected[key].shape or value.dtype!=expected[key].dtype: raise ValueError('Extension shape/dtype mismatch: '+key) if not torch.isfinite(value).all():raise ValueError('Nonfinite extension: '+key) slow.load_state_dict(state,strict=True) def load_initial(path,slow,*,expected_sha256): if sha(path)!=expected_sha256:raise ValueError('Initial artifact hash mismatch') z=torch.load(path,map_location='cpu',weights_only=True) # Explicit map for the saved pre-training container, not smoke or trained checkpoints. if z.get('schema') is not None or z.get('not_smoke') is not True or set(z).difference( {'slow','rng','parameter_names','source_initial','source_step0','source_step0_sha256','not_smoke'}): raise ValueError('Expected the reviewed pre-training initial container') state=z['slow'] if list(slow.ttt)==['17']: state={k:v for k,v in state.items() if k.startswith('ttt.17.') or k=='registers.initial'} strict_slow_load(slow,state) return z def load_weights(path,slow,config,*,expected_sha256): if '.partial.' in Path(path).name:raise ValueError('Uncommitted partial file') if sha(path)!=expected_sha256:raise ValueError('Checkpoint hash mismatch') z=torch.load(path,map_location='cpu',weights_only=True) if z.get('schema') not in LEGACY|{SCHEMA}:raise ValueError('Unsupported checkpoint schema') if z['active_layers']!=list(config.active_layers) or z['register_count']!=config.register_count: raise ValueError('Checkpoint layer/register contract mismatch') source_learned=z['schema']=='learned_inner_lr_v1' if z['schema']!=SCHEMA else z['config']['learn_inner_lr'] if source_learned!=config.learn_inner_lr:raise ValueError('Fixed/learned checkpoint mode mismatch') # Legacy slow keys already have exact ttt.{layer}.{name} / registers.initial names. strict_slow_load(slow,z['slow']) expected=[n for n,_ in slow.named_parameters() if config.learn_inner_lr or not n.endswith('.raw_lr')] if z['parameter_names']!=expected:raise ValueError('Checkpoint optimizer name/order mismatch') return z def save_boundary(directory,*,step,slow,optimizer,config,provenance,attempted,completed): if type(step) is not int or step<0:raise ValueError('Invalid completed step') names=[n for n,p in slow.named_parameters() if p.requires_grad] expected=[id(p) for p in slow.parameters() if p.requires_grad] if [id(p) for g in optimizer.param_groups for p in g['params']]!=expected:raise ValueError('Optimizer membership/order mismatch') data={'schema':SCHEMA,'active_layers':list(config.active_layers),'register_count':16, 'step':step,'next_schedule_index':step,'slow':cpu_tree(slow.state_dict()), 'optimizer':cpu_tree(optimizer.state_dict()),'parameter_names':names,'rng':capture_rng(), 'config':config.to_dict(),'provenance':copy.deepcopy(provenance), 'attempted':dict(attempted),'completed':dict(completed),'runtime_state':'fresh W0 per sequence'} directory=Path(directory);directory.mkdir(parents=True,exist_ok=True) temp=directory/f'boundary.{time.time_ns()}.pt' atomic_tensor(temp,data) if digest(torch.load(temp,map_location='cpu',weights_only=True))!=digest(data):raise IOError('Checkpoint verification failed') latest=directory/'latest.pt';previous=directory/'previous.pt' if latest.exists():os.replace(latest,previous) os.replace(temp,latest);fsync_dir(directory) return latest,data def restore_boundary(path,*,slow,optimizer,config,provenance): z=load_weights(path,slow,config,expected_sha256=sha(path)) if z['schema']!=SCHEMA or z['config']!=config.to_dict() or z['provenance']!=provenance or z['next_schedule_index']!=z['step']: raise ValueError('Resume requires matching release schema/config/provenance') expected=[id(p) for p in slow.parameters() if p.requires_grad] if [id(p) for g in optimizer.param_groups for p in g['params']]!=expected:raise ValueError('Resume optimizer mapping') optimizer.load_state_dict(z['optimizer']);restore_rng(z['rng']) if digest(optimizer.state_dict())!=digest(z['optimizer']):raise ValueError('Optimizer restore mismatch') return z def resume_step(directory,ledger): latest=Path(directory)/'latest.pt' step=torch.load(latest,map_location='cpu',weights_only=True)['step'] if latest.exists() else 0 # Never replay an optimizer attempt whose durable boundary is absent. events=[json.loads(x) for x in ledger.events.read_text().splitlines()] if ledger.events.exists() else [] for e in events: if e['event']=='reserve' and e.get('kind')=='optimizer_update' and int(e['scope'])>step: raise RuntimeError('Ambiguous optimizer execution: inspect checkpoint/ledger before any retry') return step