File size: 3,173 Bytes
0f5d220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
"""Shared S4/S16 and P/R orchestration around the unchanged sequence trainer."""
import math
import torch
from .sequence_trainer import SequenceTrainer, full_masks
from .layer_state import packed
from .checkpoint import cpu_tree

def training_masks(config,device):
    masks=full_masks(1,config.sequence_length,config.horizon,device)
    masks.action_dimension=(torch.arange(config.action_dim,device=device)<config.loss_action_dim)[None]
    if config.state_policy=='reset':masks.reset[:]=True
    return masks

def train_sequence(adapter,config,prepared,actions,epsilon,times,*,backward,after_segment=None):
    if actions.shape!=(1,config.sequence_length,config.horizon,config.action_dim):raise ValueError('Training shape differs from experiment config')
    if getattr(adapter,'use_batched_segments',False):
        from .batched_trainer import run
        return run(adapter,config,prepared,actions,epsilon,times,training_masks(config,actions.device),backward=backward,after_segment=after_segment)
    return SequenceTrainer(adapter).run(prepared,actions,epsilon,times,
        training_masks(config,actions.device),segment_length=config.segment_length,
        backward=backward,after_segment=after_segment)

def check_gradients(slow):
    out={}
    for name,p in slow.named_parameters():
        if not p.requires_grad:
            if p.grad is not None:raise RuntimeError('Frozen parameter has grad: '+name)
            continue
        if p.grad is None:raise RuntimeError('Disconnected parameter: '+name)
        if not torch.isfinite(p.grad).all():raise FloatingPointError('Nonfinite gradient: '+name)
        out[name]=float(p.grad.detach().double().norm())
    return out

class SafetyGuard:
    def __init__(self,*,device='cuda:0',max_bytes=20*2**30,max_fast=1e4,lr_bounds=(1e-6,1e-3)):
        self.device=device;self.max_bytes=max_bytes;self.max_fast=max_fast;self.lr_bounds=lr_bounds
    def __call__(self,state=None):
        if torch.device(self.device).type=='cuda' and torch.cuda.max_memory_allocated(self.device)>self.max_bytes:
            raise MemoryError('Allocated-memory guard exceeded')
        if state:
            for s in state.values():
                for w in s.weights:
                    if not torch.isfinite(w).all() or float(w.detach().abs().max())>self.max_fast:raise FloatingPointError('Fast-state guard')
    def learning_rates(self,modules):
        for layer in modules.values():
            value=float(layer.inner_lr.detach())
            if not math.isfinite(value) or not self.lr_bounds[0]<=value<=self.lr_bounds[1]:raise FloatingPointError('Inner-LR guard')

def generate_chunk(adapter,prepared,noise,incoming):
    """The model computation receives no target, episode ID, split or source index."""
    with torch.no_grad():prediction,candidate,trace=adapter.rollout(prepared,noise.clone(),incoming,differentiable=False)
    if prediction.shape!=(noise.shape[0],10,32) or not torch.isfinite(prediction).all():raise ValueError('Prediction shape/finite')
    if candidate:
        if trace['commits']!=1 or trace['whole_vector_from_iteration']!=9:raise RuntimeError('Commit contract')
    return prediction,candidate,trace