File size: 9,840 Bytes
0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | """All-timestep flow supervision; segment-global normalization; no optimizer here."""
from dataclasses import dataclass
import gc, weakref
import torch
from .core import FastState
from .layer_state import packed, clone
@dataclass
class Masks:
observation: torch.Tensor # B,T; valid observation may write with no loss
action_slot: torch.Tensor # B,T,H; not action-dimension padding
supervision: torch.Tensor # B,T
reset: torch.Tensor # B,T; episode boundary, not TBPTT boundary
action_dimension: torch.Tensor | None = None # B,D; excludes padded dimensions
def validate(self,shape):
b,t,h,d=shape
for name,expected in [('observation',(b,t)),('action_slot',(b,t,h)),('supervision',(b,t)),('reset',(b,t))]:
a=getattr(self,name)
if a.shape!=expected or a.dtype!=torch.bool:raise ValueError(name+' shape/dtype')
if bool((self.reset & ~self.observation).any()):raise ValueError('Reset on padding is invalid')
if self.action_dimension is not None and (self.action_dimension.shape!=(b,d) or self.action_dimension.dtype!=torch.bool):raise ValueError('action dimension shape/dtype')
return self
def scalar_mask(self,shape):
self.validate(shape)
valid=(self.observation[...,None]&self.action_slot&self.supervision[...,None])[...,None].expand(shape)
return valid if self.action_dimension is None else valid & self.action_dimension[:,None,None,:]
def count(self,shape):return int(self.scalar_mask(shape).sum())
def full_masks(b,t,h,device='cpu'):
valid=torch.ones((b,t),dtype=torch.bool,device=device);reset=torch.zeros_like(valid);reset[:,0]=True
return Masks(valid,torch.ones((b,t,h),dtype=torch.bool,device=device),valid.clone(),reset)
def bridge(actions,epsilon,time,masks):
if actions.ndim!=4 or actions.shape!=epsilon.shape or actions.dtype!=epsilon.dtype:raise ValueError('Actions/noise must match [B,T,H,D]')
if time.shape!=(*actions.shape[:2],1,1) or time.dtype!=actions.dtype:raise ValueError('Independent per-time [B,T,1,1] required')
masks.validate(actions.shape)
valid=masks.observation[...,None]&masks.action_slot
a=torch.where(valid[...,None],actions,0);e=torch.where(valid[...,None],epsilon,0)
s=torch.where(masks.observation[...,None,None],time,0)
if not bool(torch.isfinite(a).all() and torch.isfinite(e).all() and torch.isfinite(s).all() and (s>=0).all() and (s<=1).all()):raise ValueError('Nonfinite/range input')
return s*e+(1-s)*a,e-a
def masked_sse(prediction,target,valid):
if prediction.shape!=target.shape or valid.shape!=prediction.shape or valid.dtype!=torch.bool:raise ValueError('SSE shapes')
d=torch.where(valid,prediction,0)-torch.where(valid,target,0)
if not bool(torch.isfinite(d).all()):raise FloatingPointError('Nonfinite supervised residual')
return d.square().sum()
def commit_valid(incoming,candidate,valid):
"""Batch slot/state validity. All-False is a true no-op with no W0 bypass."""
if not bool(valid.any()):return incoming
if bool(valid.all()):return candidate
return {k:FastState(tuple(torch.where(valid.reshape(-1,*([1]*(a.ndim-1))),a,b)
for a,b in zip(candidate[k].weights,s.weights)),torch.where(valid,candidate[k].updates,s.updates)) for k,s in incoming.items()}
def _norm(p):return None if p.grad is None else float(p.grad.detach().double().norm())
class SequenceTrainer:
"""Both L1/L4 use this API. GPU validation is explicitly B1/full-valid only.
Segment graph refs are private and dropped after backward. Outputs/states in
records are detached CPU tensors; teacher-forced state is never inference input.
"""
def __init__(self,adapter):self.adapter=adapter
def _segment(self,prepared,x,u,times,masks,lo,hi,state,denom,*,differentiate,probes):
ad=self.adapter;loss=None;predictions=[];per_time=[];traces=[];refs=[];terms=[]
valid_all=masks.scalar_mask(x.shape)
for t in range(lo,hi):
valid=masks.observation[:,t]
if not bool(valid.any()):
pred=torch.zeros_like(x[:,t]);tr=[]
else:
if not bool(valid.all()) and getattr(ad,'GPU_full_valid_only',False):raise NotImplementedError('Mixed valid GPU batch is not validated')
if bool(masks.reset[:,t].any()):state=ad.reset_state(state,masks.reset[:,t])
tr=None if getattr(ad,'minimal_diagnostics',False) else []
pred,candidate=ad.velocity(prepared[t],x[:,t],times[:,t].reshape(x.shape[0]),state,
differentiable=differentiate,trace=tr,register_probe=None if probes is None else probes[t])
state=commit_valid(state,candidate,valid)
refs.append(weakref.ref(pred));refs.extend(weakref.ref(w) for s in state.values() for w in s.weights)
sse=masked_sse(pred,u[:,t],valid_all[:,t]);term=sse/max(denom,1)
loss=term if loss is None else loss+term
terms.append(term)
predictions.append(pred.detach().cpu().clone())
per_time.append({'t':t,'supervised_scalar_count':int(valid_all[:,t].sum()),'SSE':float(sse.detach()),'global_normalized_loss':float(term.detach()),'updates':{k:s.updates.tolist() for k,s in state.items()}})
traces.append(tr)
return loss,state,predictions,per_time,traces,refs,terms
def run(self,prepared,actions,epsilon,times,masks,*,segment_length=None,backward=None,probes=None,before_backward=None,after_segment=None):
b,t,h,d=actions.shape
if len(prepared)!=t:raise ValueError('One observation per timestep')
if getattr(self.adapter,'GPU_full_valid_only',False):
if b!=1 or not bool(masks.observation.all() and masks.action_slot.all()):raise NotImplementedError('VLA validation is full-valid B1 only')
x,u=bridge(actions,epsilon,times,masks);denom=masks.count(actions.shape)
S=t if segment_length is None else segment_length
if type(S)is not int or S<1:raise ValueError('Positive segment size')
state=self.adapter.initial_state(b);out=[];records=[];per=[];alltraces=[];logged=0.0
for lo in range(0,t,S):
hi=min(t,lo+S)
with torch.enable_grad() if backward is not None else torch.no_grad():
loss,graph_state,pp,tt,tr,refs,terms=self._segment(prepared,x,u,times,masks,lo,hi,state,denom,differentiate=backward is not None,probes=probes)
# Detach before backward, retaining values/counters and freeing segment graph after it.
detached=self.adapter.detach_state(graph_state)
assert all(torch.equal(a,b) for k in state for a,b in zip(graph_state[k].weights,detached[k].weights))
assert all(torch.equal(graph_state[k].updates,detached[k].updates) for k in state)
assert all(w.grad_fn is None and not w.requires_grad for s in detached.values() for w in s.weights)
rec={'start':lo,'end':hi,'denominator':denom,'loss':float(loss.detach()),'boundary_detach':hi<t,
'state_values_counters_preserved':True,'probes':None}
if backward is not None and sum(r['supervised_scalar_count'] for r in tt) and loss.requires_grad:
if probes is not None:
for p in probes:p.grad=None # inner/probe accumulation is not outer evidence
if before_backward is not None:before_backward(loss,lo,hi,rec,terms)
backward(loss)
if probes is not None:rec['probes']=[_norm(p) for p in probes]
logged+=rec['loss'];out.extend(pp);per.extend(tt);alltraces.extend(tr)
state=detached
del loss,graph_state,detached,pp,tt,tr,terms
gc.collect()
rec['segment_prediction_state_python_refs_alive']=sum(r() is not None for r in refs)
del refs
if after_segment is not None:after_segment(rec)
records.append(rec)
return {'prediction':torch.stack(out,dim=1),'loss':logged,'valid_scalar_count':denom,
'state':packed(state),'segments':records,'per_time':per,'traces':alltraces,
'history_type':'teacher_forced_flow','optimizer_steps_inside_trainer':0}
def sequential_generate(adapter,prepared,noise,*,reset_each=False,on_chunk=None):
"""Target-free API: accepts no actions, supervision mask, or flow runtime state."""
if noise.ndim!=4 or noise.shape[1]!=len(prepared):raise ValueError('Noise [B,T,H,D]')
state=adapter.initial_state(noise.shape[0]);predictions=[];records=[]
with torch.no_grad():
for t,obs in enumerate(prepared):
if reset_each:state=adapter.initial_state(noise.shape[0])
before=packed(state)
if on_chunk:on_chunk('attempt',t)
pred,candidate,rec=adapter.rollout(obs,noise[:,t].clone(),state,differentiable=False)
if on_chunk:on_chunk('complete',t)
assert rec['commits']==1 and rec['whole_vector_from_iteration']==9
state=candidate
assert all(s.updates.tolist()==[1 if reset_each else t+1]*noise.shape[0] for s in state.values())
predictions.append(pred.detach().cpu().clone());records.append({'t':t,'incoming':before,'committed':packed(state),'solver':rec})
return {'prediction':torch.stack(predictions,dim=1),'state':packed(state),'records':records,
'history_type':'target_free_sequential','reset_each':reset_each,'target_in_API':False}
def score_actions(predictions,targets):
if predictions.shape!=targets.shape:raise ValueError('Scoring shape')
e=(predictions.detach().cpu().double()-targets.detach().cpu().double()).square()
return {name:float(e[...,sl].mean()) for name,sl in [('real7',slice(0,7)),('padding25',slice(7,32)),('full32',slice(None))]}
|