jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
5.37 kB
"""Strict layer-keyed state operations. No VLA, optimizer, or CUDA initialization."""
from dataclasses import asdict
import torch
from .core import FastState, TTTConfig, TTTLayer
from .eval_io import atomic_tensor
def layer_ids(indices, depth):
if not isinstance(indices, (list, tuple)) or not indices:
raise ValueError('A nonempty ordered layer list is required')
if any(type(i) is not int or not 0 <= i < depth for i in indices):
raise ValueError('Invalid zero-based block index')
if list(indices) != sorted(set(indices)):
raise ValueError('Layers must be unique and in transformer order')
return tuple(indices)
def validate(modules, state, batch=None):
if not isinstance(state, dict) or set(state) != set(modules):
raise ValueError('State keys must exactly match active layers')
stores = set()
for key, m in modules.items():
s = state[key]
if not isinstance(s, FastState): raise TypeError('Expected FastState')
if s.updates.ndim != 1 or s.updates.dtype != torch.long:
raise ValueError('Update counter must be int64 [B]')
b = s.updates.numel()
if b < 1 or (batch is not None and b != batch): raise ValueError('Batch mismatch')
if s.updates.device != m.alpha.device or bool((s.updates < 0).any()):
raise ValueError('Counter device/value mismatch')
if len(s.weights) != len(m.w0): raise ValueError('Weight tuple mismatch')
for w, w0 in zip(s.weights, m.w0):
if w.shape != (b, *w0.shape) or w.dtype != w0.dtype or w.device != w0.device:
raise ValueError('Fast weight shape/dtype/device mismatch')
for w in (*s.weights, s.updates):
ptr = (str(w.device), w.untyped_storage().data_ptr())
if ptr in stores: raise ValueError('Fast-state storage alias')
stores.add(ptr)
return state
def initial(modules, batch=1):
return validate(modules, {k:m.initial_state(batch) for k,m in modules.items()}, batch)
def clone(state, *, detach=False):
def copy(x): return (x.detach() if detach else x).clone()
return {k:FastState(tuple(copy(w) for w in s.weights), copy(s.updates)) for k,s in state.items()}
def detach_all(modules, state):
validate(modules,state)
return validate(modules,clone(state,detach=True))
def reset(modules, state, mask, *, selected=None):
validate(modules,state)
keys = list(modules) if selected is None else list(selected)
if len(keys)!=len(set(keys)) or not set(keys)<=set(modules): raise ValueError('Invalid reset layer keys')
b=next(iter(state.values())).updates.numel()
if mask.shape!=(b,) or mask.dtype!=torch.bool or mask.device!=next(iter(state.values())).updates.device:
raise ValueError('Reset mask must be bool [B] on state device')
# Preserve the existing true no-op, including absence of a W0 bypass path.
if not keys or not bool(mask.any()): return state
out=clone(state)
for k in keys:
# A complete reset has no old-state graph, unlike where(True, W0, old).
out[k]=modules[k].initial_state(b) if bool(mask.all()) else modules[k].reset(state[k],mask)
return validate(modules,out)
def packed(state):
return {k:{'weights':tuple(w.detach().cpu().clone() for w in s.weights),
'updates':s.updates.detach().cpu().clone()} for k,s in state.items()}
def unpacked(data, *, device='cpu'):
return {k:FastState(tuple(w.to(device).clone() for w in s['weights']),s['updates'].to(device).clone()) for k,s in data.items()}
def parameter_storage_unique(modules):
# Module.parameters() deduplicates shared objects; inspect each layer separately.
params=[p for m in modules.values() for p in m.parameters()]
return len({id(p) for p in params})==len(params) and len({(str(p.device),p.untyped_storage().data_ptr()) for p in params})==len(params)
def save_extension(*, path, modules, registers, depth, provenance, state=None):
ids=layer_ids([int(k) for k in modules],depth)
if state is not None: validate(modules,state)
return atomic_tensor(path,{'schema':1,'active_layers':list(ids),'depth':depth,
'ttt_config':{k:asdict(m.config) for k,m in modules.items()},
'ttt':{n:v.detach().cpu().clone() for n,v in modules.state_dict().items()},
'register_config':registers.metadata(),
'registers':{n:v.detach().cpu().clone() for n,v in registers.state_dict().items()},
'layout':{'register_span':[0,16],'action_span':[16,26],'state_span':None},
'runtime':None if state is None else packed(state),'provenance':provenance,'optimizer_created':False})
def load_extension(*, path, modules, registers, depth):
z=torch.load(path,map_location='cpu',weights_only=True)
ids=layer_ids(z['active_layers'],depth)
if list(modules)!=[str(i) for i in ids] or z['depth']!=depth: raise ValueError('Checkpoint layer mapping mismatch')
if z['register_config']!=registers.metadata(): raise ValueError('Register config mismatch')
if z['ttt_config']!={k:asdict(m.config) for k,m in modules.items()}:raise ValueError('TTT config mismatch')
modules.load_state_dict(z['ttt'],strict=True);registers.load_state_dict(z['registers'],strict=True)
if z['runtime'] is not None:validate(modules,unpacked(z['runtime'],device=next(modules.parameters()).device))
return z