jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
7.1 kB
"""Versioned extension checkpoints only; no base policy weights."""
from pathlib import Path
import copy,hashlib,json,os,random,time
import numpy as np
import torch
from .eval_io import sha,parameter_hash,tensor_hash,atomic_tensor
def cpu_tree(v):
if isinstance(v,torch.Tensor):return v.detach().cpu().clone()
if isinstance(v,dict):return {k:cpu_tree(x) for k,x in v.items()}
if isinstance(v,list):return [cpu_tree(x) for x in v]
if isinstance(v,tuple):return tuple(cpu_tree(x) for x in v)
return copy.deepcopy(v)
def digest(v):
h=hashlib.sha256()
def visit(x):
if isinstance(x,torch.Tensor):h.update(('tensor:'+tensor_hash(x)).encode())
elif isinstance(x,dict):
h.update(b'dict')
for k in sorted(x,key=lambda k:(type(k).__name__,str(k))):visit(k);visit(x[k])
elif isinstance(x,(list,tuple)):
h.update(type(x).__name__.encode())
for y in x:visit(y)
else:h.update((type(x).__name__+':'+repr(x)).encode())
visit(v);return h.hexdigest()
def capture_rng():
n=np.random.get_state()
return {'torch':torch.get_rng_state().clone(),'cuda':torch.cuda.get_rng_state(0).cpu().clone() if torch.cuda.is_initialized() else None,
'python':random.getstate(),'numpy':{'name':n[0],'keys':torch.from_numpy(n[1].astype(np.int64)),'position':n[2],'has_gauss':n[3],'cached':n[4]}}
def restore_rng(rng):
torch.set_rng_state(rng['torch']);random.setstate(rng['python']);n=rng['numpy']
np.random.set_state((n['name'],n['keys'].numpy().astype(np.uint32),n['position'],n['has_gauss'],n['cached']))
if rng['cuda'] is not None:
if not torch.cuda.is_initialized():raise RuntimeError('Saved CUDA RNG requires the single already loaded GPU instance')
torch.cuda.set_rng_state(rng['cuda'],0)
def fsync_dir(path):
fd=os.open(path,os.O_RDONLY)
try:os.fsync(fd)
finally:os.close(fd)
SCHEMA = 'robottt_pi05.extension.v1'
LEGACY = {'T16_sequence_training_v1','learned_inner_lr_v1','bptt_horizon_v1'}
def strict_slow_load(slow,state):
expected=slow.state_dict()
if set(state)!=set(expected):raise ValueError('Missing or unexpected extension keys')
for key,value in state.items():
if not isinstance(value,torch.Tensor) or value.shape!=expected[key].shape or value.dtype!=expected[key].dtype:
raise ValueError('Extension shape/dtype mismatch: '+key)
if not torch.isfinite(value).all():raise ValueError('Nonfinite extension: '+key)
slow.load_state_dict(state,strict=True)
def load_initial(path,slow,*,expected_sha256):
if sha(path)!=expected_sha256:raise ValueError('Initial artifact hash mismatch')
z=torch.load(path,map_location='cpu',weights_only=True)
# Explicit map for the saved pre-training container, not smoke or trained checkpoints.
if z.get('schema') is not None or z.get('not_smoke') is not True or set(z).difference(
{'slow','rng','parameter_names','source_initial','source_step0','source_step0_sha256','not_smoke'}):
raise ValueError('Expected the reviewed pre-training initial container')
state=z['slow']
if list(slow.ttt)==['17']:
state={k:v for k,v in state.items() if k.startswith('ttt.17.') or k=='registers.initial'}
strict_slow_load(slow,state)
return z
def load_weights(path,slow,config,*,expected_sha256):
if '.partial.' in Path(path).name:raise ValueError('Uncommitted partial file')
if sha(path)!=expected_sha256:raise ValueError('Checkpoint hash mismatch')
z=torch.load(path,map_location='cpu',weights_only=True)
if z.get('schema') not in LEGACY|{SCHEMA}:raise ValueError('Unsupported checkpoint schema')
if z['active_layers']!=list(config.active_layers) or z['register_count']!=config.register_count:
raise ValueError('Checkpoint layer/register contract mismatch')
source_learned=z['schema']=='learned_inner_lr_v1' if z['schema']!=SCHEMA else z['config']['learn_inner_lr']
if source_learned!=config.learn_inner_lr:raise ValueError('Fixed/learned checkpoint mode mismatch')
# Legacy slow keys already have exact ttt.{layer}.{name} / registers.initial names.
strict_slow_load(slow,z['slow'])
expected=[n for n,_ in slow.named_parameters() if config.learn_inner_lr or not n.endswith('.raw_lr')]
if z['parameter_names']!=expected:raise ValueError('Checkpoint optimizer name/order mismatch')
return z
def save_boundary(directory,*,step,slow,optimizer,config,provenance,attempted,completed):
if type(step) is not int or step<0:raise ValueError('Invalid completed step')
names=[n for n,p in slow.named_parameters() if p.requires_grad]
expected=[id(p) for p in slow.parameters() if p.requires_grad]
if [id(p) for g in optimizer.param_groups for p in g['params']]!=expected:raise ValueError('Optimizer membership/order mismatch')
data={'schema':SCHEMA,'active_layers':list(config.active_layers),'register_count':16,
'step':step,'next_schedule_index':step,'slow':cpu_tree(slow.state_dict()),
'optimizer':cpu_tree(optimizer.state_dict()),'parameter_names':names,'rng':capture_rng(),
'config':config.to_dict(),'provenance':copy.deepcopy(provenance),
'attempted':dict(attempted),'completed':dict(completed),'runtime_state':'fresh W0 per sequence'}
directory=Path(directory);directory.mkdir(parents=True,exist_ok=True)
temp=directory/f'boundary.{time.time_ns()}.pt'
atomic_tensor(temp,data)
if digest(torch.load(temp,map_location='cpu',weights_only=True))!=digest(data):raise IOError('Checkpoint verification failed')
latest=directory/'latest.pt';previous=directory/'previous.pt'
if latest.exists():os.replace(latest,previous)
os.replace(temp,latest);fsync_dir(directory)
return latest,data
def restore_boundary(path,*,slow,optimizer,config,provenance):
z=load_weights(path,slow,config,expected_sha256=sha(path))
if z['schema']!=SCHEMA or z['config']!=config.to_dict() or z['provenance']!=provenance or z['next_schedule_index']!=z['step']:
raise ValueError('Resume requires matching release schema/config/provenance')
expected=[id(p) for p in slow.parameters() if p.requires_grad]
if [id(p) for g in optimizer.param_groups for p in g['params']]!=expected:raise ValueError('Resume optimizer mapping')
optimizer.load_state_dict(z['optimizer']);restore_rng(z['rng'])
if digest(optimizer.state_dict())!=digest(z['optimizer']):raise ValueError('Optimizer restore mismatch')
return z
def resume_step(directory,ledger):
latest=Path(directory)/'latest.pt'
step=torch.load(latest,map_location='cpu',weights_only=True)['step'] if latest.exists() else 0
# Never replay an optimizer attempt whose durable boundary is absent.
events=[json.loads(x) for x in ledger.events.read_text().splitlines()] if ledger.events.exists() else []
for e in events:
if e['event']=='reserve' and e.get('kind')=='optimizer_update' and int(e['scope'])>step:
raise RuntimeError('Ambiguous optimizer execution: inspect checkpoint/ledger before any retry')
return step