File size: 9,840 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
"""All-timestep flow supervision; segment-global normalization; no optimizer here."""
from dataclasses import dataclass
import gc, weakref
import torch
from .core import FastState
from .layer_state import packed, clone

@dataclass
class Masks:
    observation: torch.Tensor       # B,T; valid observation may write with no loss
    action_slot: torch.Tensor       # B,T,H; not action-dimension padding
    supervision: torch.Tensor       # B,T
    reset: torch.Tensor             # B,T; episode boundary, not TBPTT boundary

    action_dimension: torch.Tensor | None = None  # B,D; excludes padded dimensions

    def validate(self,shape):
        b,t,h,d=shape
        for name,expected in [('observation',(b,t)),('action_slot',(b,t,h)),('supervision',(b,t)),('reset',(b,t))]:
            a=getattr(self,name)
            if a.shape!=expected or a.dtype!=torch.bool:raise ValueError(name+' shape/dtype')
        if bool((self.reset & ~self.observation).any()):raise ValueError('Reset on padding is invalid')
        if self.action_dimension is not None and (self.action_dimension.shape!=(b,d) or self.action_dimension.dtype!=torch.bool):raise ValueError('action dimension shape/dtype')
        return self

    def scalar_mask(self,shape):
        self.validate(shape)
        valid=(self.observation[...,None]&self.action_slot&self.supervision[...,None])[...,None].expand(shape)
        return valid if self.action_dimension is None else valid & self.action_dimension[:,None,None,:]

    def count(self,shape):return int(self.scalar_mask(shape).sum())

def full_masks(b,t,h,device='cpu'):
    valid=torch.ones((b,t),dtype=torch.bool,device=device);reset=torch.zeros_like(valid);reset[:,0]=True
    return Masks(valid,torch.ones((b,t,h),dtype=torch.bool,device=device),valid.clone(),reset)

def bridge(actions,epsilon,time,masks):
    if actions.ndim!=4 or actions.shape!=epsilon.shape or actions.dtype!=epsilon.dtype:raise ValueError('Actions/noise must match [B,T,H,D]')
    if time.shape!=(*actions.shape[:2],1,1) or time.dtype!=actions.dtype:raise ValueError('Independent per-time [B,T,1,1] required')
    masks.validate(actions.shape)
    valid=masks.observation[...,None]&masks.action_slot
    a=torch.where(valid[...,None],actions,0);e=torch.where(valid[...,None],epsilon,0)
    s=torch.where(masks.observation[...,None,None],time,0)
    if not bool(torch.isfinite(a).all() and torch.isfinite(e).all() and torch.isfinite(s).all() and (s>=0).all() and (s<=1).all()):raise ValueError('Nonfinite/range input')
    return s*e+(1-s)*a,e-a

def masked_sse(prediction,target,valid):
    if prediction.shape!=target.shape or valid.shape!=prediction.shape or valid.dtype!=torch.bool:raise ValueError('SSE shapes')
    d=torch.where(valid,prediction,0)-torch.where(valid,target,0)
    if not bool(torch.isfinite(d).all()):raise FloatingPointError('Nonfinite supervised residual')
    return d.square().sum()

def commit_valid(incoming,candidate,valid):
    """Batch slot/state validity. All-False is a true no-op with no W0 bypass."""
    if not bool(valid.any()):return incoming
    if bool(valid.all()):return candidate
    return {k:FastState(tuple(torch.where(valid.reshape(-1,*([1]*(a.ndim-1))),a,b)
           for a,b in zip(candidate[k].weights,s.weights)),torch.where(valid,candidate[k].updates,s.updates)) for k,s in incoming.items()}

def _norm(p):return None if p.grad is None else float(p.grad.detach().double().norm())

class SequenceTrainer:
    """Both L1/L4 use this API. GPU validation is explicitly B1/full-valid only.

    Segment graph refs are private and dropped after backward. Outputs/states in
    records are detached CPU tensors; teacher-forced state is never inference input.
    """
    def __init__(self,adapter):self.adapter=adapter

    def _segment(self,prepared,x,u,times,masks,lo,hi,state,denom,*,differentiate,probes):
        ad=self.adapter;loss=None;predictions=[];per_time=[];traces=[];refs=[];terms=[]
        valid_all=masks.scalar_mask(x.shape)
        for t in range(lo,hi):
            valid=masks.observation[:,t]
            if not bool(valid.any()):
                pred=torch.zeros_like(x[:,t]);tr=[]
            else:
                if not bool(valid.all()) and getattr(ad,'GPU_full_valid_only',False):raise NotImplementedError('Mixed valid GPU batch is not validated')
                if bool(masks.reset[:,t].any()):state=ad.reset_state(state,masks.reset[:,t])
                tr=None if getattr(ad,'minimal_diagnostics',False) else []
                pred,candidate=ad.velocity(prepared[t],x[:,t],times[:,t].reshape(x.shape[0]),state,
                    differentiable=differentiate,trace=tr,register_probe=None if probes is None else probes[t])
                state=commit_valid(state,candidate,valid)
                refs.append(weakref.ref(pred));refs.extend(weakref.ref(w) for s in state.values() for w in s.weights)
            sse=masked_sse(pred,u[:,t],valid_all[:,t]);term=sse/max(denom,1)
            loss=term if loss is None else loss+term
            terms.append(term)
            predictions.append(pred.detach().cpu().clone())
            per_time.append({'t':t,'supervised_scalar_count':int(valid_all[:,t].sum()),'SSE':float(sse.detach()),'global_normalized_loss':float(term.detach()),'updates':{k:s.updates.tolist() for k,s in state.items()}})
            traces.append(tr)
        return loss,state,predictions,per_time,traces,refs,terms

    def run(self,prepared,actions,epsilon,times,masks,*,segment_length=None,backward=None,probes=None,before_backward=None,after_segment=None):
        b,t,h,d=actions.shape
        if len(prepared)!=t:raise ValueError('One observation per timestep')
        if getattr(self.adapter,'GPU_full_valid_only',False):
            if b!=1 or not bool(masks.observation.all() and masks.action_slot.all()):raise NotImplementedError('VLA validation is full-valid B1 only')
        x,u=bridge(actions,epsilon,times,masks);denom=masks.count(actions.shape)
        S=t if segment_length is None else segment_length
        if type(S)is not int or S<1:raise ValueError('Positive segment size')
        state=self.adapter.initial_state(b);out=[];records=[];per=[];alltraces=[];logged=0.0
        for lo in range(0,t,S):
            hi=min(t,lo+S)
            with torch.enable_grad() if backward is not None else torch.no_grad():
                loss,graph_state,pp,tt,tr,refs,terms=self._segment(prepared,x,u,times,masks,lo,hi,state,denom,differentiate=backward is not None,probes=probes)
            # Detach before backward, retaining values/counters and freeing segment graph after it.
            detached=self.adapter.detach_state(graph_state)
            assert all(torch.equal(a,b) for k in state for a,b in zip(graph_state[k].weights,detached[k].weights))
            assert all(torch.equal(graph_state[k].updates,detached[k].updates) for k in state)
            assert all(w.grad_fn is None and not w.requires_grad for s in detached.values() for w in s.weights)
            rec={'start':lo,'end':hi,'denominator':denom,'loss':float(loss.detach()),'boundary_detach':hi<t,
                 'state_values_counters_preserved':True,'probes':None}
            if backward is not None and sum(r['supervised_scalar_count'] for r in tt) and loss.requires_grad:
                if probes is not None:
                    for p in probes:p.grad=None  # inner/probe accumulation is not outer evidence
                if before_backward is not None:before_backward(loss,lo,hi,rec,terms)
                backward(loss)
                if probes is not None:rec['probes']=[_norm(p) for p in probes]
            logged+=rec['loss'];out.extend(pp);per.extend(tt);alltraces.extend(tr)
            state=detached
            del loss,graph_state,detached,pp,tt,tr,terms
            gc.collect()
            rec['segment_prediction_state_python_refs_alive']=sum(r() is not None for r in refs)
            del refs
            if after_segment is not None:after_segment(rec)
            records.append(rec)
        return {'prediction':torch.stack(out,dim=1),'loss':logged,'valid_scalar_count':denom,
            'state':packed(state),'segments':records,'per_time':per,'traces':alltraces,
            'history_type':'teacher_forced_flow','optimizer_steps_inside_trainer':0}

def sequential_generate(adapter,prepared,noise,*,reset_each=False,on_chunk=None):
    """Target-free API: accepts no actions, supervision mask, or flow runtime state."""
    if noise.ndim!=4 or noise.shape[1]!=len(prepared):raise ValueError('Noise [B,T,H,D]')
    state=adapter.initial_state(noise.shape[0]);predictions=[];records=[]
    with torch.no_grad():
        for t,obs in enumerate(prepared):
            if reset_each:state=adapter.initial_state(noise.shape[0])
            before=packed(state)
            if on_chunk:on_chunk('attempt',t)
            pred,candidate,rec=adapter.rollout(obs,noise[:,t].clone(),state,differentiable=False)
            if on_chunk:on_chunk('complete',t)
            assert rec['commits']==1 and rec['whole_vector_from_iteration']==9
            state=candidate
            assert all(s.updates.tolist()==[1 if reset_each else t+1]*noise.shape[0] for s in state.values())
            predictions.append(pred.detach().cpu().clone());records.append({'t':t,'incoming':before,'committed':packed(state),'solver':rec})
    return {'prediction':torch.stack(predictions,dim=1),'state':packed(state),'records':records,
            'history_type':'target_free_sequential','reset_each':reset_each,'target_in_API':False}

def score_actions(predictions,targets):
    if predictions.shape!=targets.shape:raise ValueError('Scoring shape')
    e=(predictions.detach().cpu().double()-targets.detach().cpu().double()).square()
    return {name:float(e[...,sl].mean()) for name,sl in [('real7',slice(0,7)),('padding25',slice(7,32)),('full32',slice(None))]}