jisue's picture
Experiment backup 2026-11-10T01:58:18.622752+00:00
0f5d220 verified
Raw History Blame Contribute Delete
9.84 kB
"""All-timestep flow supervision; segment-global normalization; no optimizer here."""
from dataclasses import dataclass
import gc, weakref
import torch
from .core import FastState
from .layer_state import packed, clone
@dataclass
class Masks:
observation: torch.Tensor # B,T; valid observation may write with no loss
action_slot: torch.Tensor # B,T,H; not action-dimension padding
supervision: torch.Tensor # B,T
reset: torch.Tensor # B,T; episode boundary, not TBPTT boundary
action_dimension: torch.Tensor | None = None # B,D; excludes padded dimensions
def validate(self,shape):
b,t,h,d=shape
for name,expected in [('observation',(b,t)),('action_slot',(b,t,h)),('supervision',(b,t)),('reset',(b,t))]:
a=getattr(self,name)
if a.shape!=expected or a.dtype!=torch.bool:raise ValueError(name+' shape/dtype')
if bool((self.reset & ~self.observation).any()):raise ValueError('Reset on padding is invalid')
if self.action_dimension is not None and (self.action_dimension.shape!=(b,d) or self.action_dimension.dtype!=torch.bool):raise ValueError('action dimension shape/dtype')
return self
def scalar_mask(self,shape):
self.validate(shape)
valid=(self.observation[...,None]&self.action_slot&self.supervision[...,None])[...,None].expand(shape)
return valid if self.action_dimension is None else valid & self.action_dimension[:,None,None,:]
def count(self,shape):return int(self.scalar_mask(shape).sum())
def full_masks(b,t,h,device='cpu'):
valid=torch.ones((b,t),dtype=torch.bool,device=device);reset=torch.zeros_like(valid);reset[:,0]=True
return Masks(valid,torch.ones((b,t,h),dtype=torch.bool,device=device),valid.clone(),reset)
def bridge(actions,epsilon,time,masks):
if actions.ndim!=4 or actions.shape!=epsilon.shape or actions.dtype!=epsilon.dtype:raise ValueError('Actions/noise must match [B,T,H,D]')
if time.shape!=(*actions.shape[:2],1,1) or time.dtype!=actions.dtype:raise ValueError('Independent per-time [B,T,1,1] required')
masks.validate(actions.shape)
valid=masks.observation[...,None]&masks.action_slot
a=torch.where(valid[...,None],actions,0);e=torch.where(valid[...,None],epsilon,0)
s=torch.where(masks.observation[...,None,None],time,0)
if not bool(torch.isfinite(a).all() and torch.isfinite(e).all() and torch.isfinite(s).all() and (s>=0).all() and (s<=1).all()):raise ValueError('Nonfinite/range input')
return s*e+(1-s)*a,e-a
def masked_sse(prediction,target,valid):
if prediction.shape!=target.shape or valid.shape!=prediction.shape or valid.dtype!=torch.bool:raise ValueError('SSE shapes')
d=torch.where(valid,prediction,0)-torch.where(valid,target,0)
if not bool(torch.isfinite(d).all()):raise FloatingPointError('Nonfinite supervised residual')
return d.square().sum()
def commit_valid(incoming,candidate,valid):
"""Batch slot/state validity. All-False is a true no-op with no W0 bypass."""
if not bool(valid.any()):return incoming
if bool(valid.all()):return candidate
return {k:FastState(tuple(torch.where(valid.reshape(-1,*([1]*(a.ndim-1))),a,b)
for a,b in zip(candidate[k].weights,s.weights)),torch.where(valid,candidate[k].updates,s.updates)) for k,s in incoming.items()}
def _norm(p):return None if p.grad is None else float(p.grad.detach().double().norm())
class SequenceTrainer:
"""Both L1/L4 use this API. GPU validation is explicitly B1/full-valid only.
Segment graph refs are private and dropped after backward. Outputs/states in
records are detached CPU tensors; teacher-forced state is never inference input.
"""
def __init__(self,adapter):self.adapter=adapter
def _segment(self,prepared,x,u,times,masks,lo,hi,state,denom,*,differentiate,probes):
ad=self.adapter;loss=None;predictions=[];per_time=[];traces=[];refs=[];terms=[]
valid_all=masks.scalar_mask(x.shape)
for t in range(lo,hi):
valid=masks.observation[:,t]
if not bool(valid.any()):
pred=torch.zeros_like(x[:,t]);tr=[]
else:
if not bool(valid.all()) and getattr(ad,'GPU_full_valid_only',False):raise NotImplementedError('Mixed valid GPU batch is not validated')
if bool(masks.reset[:,t].any()):state=ad.reset_state(state,masks.reset[:,t])
tr=None if getattr(ad,'minimal_diagnostics',False) else []
pred,candidate=ad.velocity(prepared[t],x[:,t],times[:,t].reshape(x.shape[0]),state,
differentiable=differentiate,trace=tr,register_probe=None if probes is None else probes[t])
state=commit_valid(state,candidate,valid)
refs.append(weakref.ref(pred));refs.extend(weakref.ref(w) for s in state.values() for w in s.weights)
sse=masked_sse(pred,u[:,t],valid_all[:,t]);term=sse/max(denom,1)
loss=term if loss is None else loss+term
terms.append(term)
predictions.append(pred.detach().cpu().clone())
per_time.append({'t':t,'supervised_scalar_count':int(valid_all[:,t].sum()),'SSE':float(sse.detach()),'global_normalized_loss':float(term.detach()),'updates':{k:s.updates.tolist() for k,s in state.items()}})
traces.append(tr)
return loss,state,predictions,per_time,traces,refs,terms
def run(self,prepared,actions,epsilon,times,masks,*,segment_length=None,backward=None,probes=None,before_backward=None,after_segment=None):
b,t,h,d=actions.shape
if len(prepared)!=t:raise ValueError('One observation per timestep')
if getattr(self.adapter,'GPU_full_valid_only',False):
if b!=1 or not bool(masks.observation.all() and masks.action_slot.all()):raise NotImplementedError('VLA validation is full-valid B1 only')
x,u=bridge(actions,epsilon,times,masks);denom=masks.count(actions.shape)
S=t if segment_length is None else segment_length
if type(S)is not int or S<1:raise ValueError('Positive segment size')
state=self.adapter.initial_state(b);out=[];records=[];per=[];alltraces=[];logged=0.0
for lo in range(0,t,S):
hi=min(t,lo+S)
with torch.enable_grad() if backward is not None else torch.no_grad():
loss,graph_state,pp,tt,tr,refs,terms=self._segment(prepared,x,u,times,masks,lo,hi,state,denom,differentiate=backward is not None,probes=probes)
# Detach before backward, retaining values/counters and freeing segment graph after it.
detached=self.adapter.detach_state(graph_state)
assert all(torch.equal(a,b) for k in state for a,b in zip(graph_state[k].weights,detached[k].weights))
assert all(torch.equal(graph_state[k].updates,detached[k].updates) for k in state)
assert all(w.grad_fn is None and not w.requires_grad for s in detached.values() for w in s.weights)
rec={'start':lo,'end':hi,'denominator':denom,'loss':float(loss.detach()),'boundary_detach':hi<t,
'state_values_counters_preserved':True,'probes':None}
if backward is not None and sum(r['supervised_scalar_count'] for r in tt) and loss.requires_grad:
if probes is not None:
for p in probes:p.grad=None # inner/probe accumulation is not outer evidence
if before_backward is not None:before_backward(loss,lo,hi,rec,terms)
backward(loss)
if probes is not None:rec['probes']=[_norm(p) for p in probes]
logged+=rec['loss'];out.extend(pp);per.extend(tt);alltraces.extend(tr)
state=detached
del loss,graph_state,detached,pp,tt,tr,terms
gc.collect()
rec['segment_prediction_state_python_refs_alive']=sum(r() is not None for r in refs)
del refs
if after_segment is not None:after_segment(rec)
records.append(rec)
return {'prediction':torch.stack(out,dim=1),'loss':logged,'valid_scalar_count':denom,
'state':packed(state),'segments':records,'per_time':per,'traces':alltraces,
'history_type':'teacher_forced_flow','optimizer_steps_inside_trainer':0}
def sequential_generate(adapter,prepared,noise,*,reset_each=False,on_chunk=None):
"""Target-free API: accepts no actions, supervision mask, or flow runtime state."""
if noise.ndim!=4 or noise.shape[1]!=len(prepared):raise ValueError('Noise [B,T,H,D]')
state=adapter.initial_state(noise.shape[0]);predictions=[];records=[]
with torch.no_grad():
for t,obs in enumerate(prepared):
if reset_each:state=adapter.initial_state(noise.shape[0])
before=packed(state)
if on_chunk:on_chunk('attempt',t)
pred,candidate,rec=adapter.rollout(obs,noise[:,t].clone(),state,differentiable=False)
if on_chunk:on_chunk('complete',t)
assert rec['commits']==1 and rec['whole_vector_from_iteration']==9
state=candidate
assert all(s.updates.tolist()==[1 if reset_each else t+1]*noise.shape[0] for s in state.values())
predictions.append(pred.detach().cpu().clone());records.append({'t':t,'incoming':before,'committed':packed(state),'solver':rec})
return {'prediction':torch.stack(predictions,dim=1),'state':packed(state),'records':records,
'history_type':'target_free_sequential','reset_each':reset_each,'target_in_API':False}
def score_actions(predictions,targets):
if predictions.shape!=targets.shape:raise ValueError('Scoring shape')
e=(predictions.detach().cpu().double()-targets.detach().cpu().double()).square()
return {name:float(e[...,sl].mean()) for name,sl in [('real7',slice(0,7)),('padding25',slice(7,32)),('full32',slice(None))]}