File size: 3,173 Bytes
0f5d220 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Shared S4/S16 and P/R orchestration around the unchanged sequence trainer."""
import math
import torch
from .sequence_trainer import SequenceTrainer, full_masks
from .layer_state import packed
from .checkpoint import cpu_tree
def training_masks(config,device):
masks=full_masks(1,config.sequence_length,config.horizon,device)
masks.action_dimension=(torch.arange(config.action_dim,device=device)<config.loss_action_dim)[None]
if config.state_policy=='reset':masks.reset[:]=True
return masks
def train_sequence(adapter,config,prepared,actions,epsilon,times,*,backward,after_segment=None):
if actions.shape!=(1,config.sequence_length,config.horizon,config.action_dim):raise ValueError('Training shape differs from experiment config')
if getattr(adapter,'use_batched_segments',False):
from .batched_trainer import run
return run(adapter,config,prepared,actions,epsilon,times,training_masks(config,actions.device),backward=backward,after_segment=after_segment)
return SequenceTrainer(adapter).run(prepared,actions,epsilon,times,
training_masks(config,actions.device),segment_length=config.segment_length,
backward=backward,after_segment=after_segment)
def check_gradients(slow):
out={}
for name,p in slow.named_parameters():
if not p.requires_grad:
if p.grad is not None:raise RuntimeError('Frozen parameter has grad: '+name)
continue
if p.grad is None:raise RuntimeError('Disconnected parameter: '+name)
if not torch.isfinite(p.grad).all():raise FloatingPointError('Nonfinite gradient: '+name)
out[name]=float(p.grad.detach().double().norm())
return out
class SafetyGuard:
def __init__(self,*,device='cuda:0',max_bytes=20*2**30,max_fast=1e4,lr_bounds=(1e-6,1e-3)):
self.device=device;self.max_bytes=max_bytes;self.max_fast=max_fast;self.lr_bounds=lr_bounds
def __call__(self,state=None):
if torch.device(self.device).type=='cuda' and torch.cuda.max_memory_allocated(self.device)>self.max_bytes:
raise MemoryError('Allocated-memory guard exceeded')
if state:
for s in state.values():
for w in s.weights:
if not torch.isfinite(w).all() or float(w.detach().abs().max())>self.max_fast:raise FloatingPointError('Fast-state guard')
def learning_rates(self,modules):
for layer in modules.values():
value=float(layer.inner_lr.detach())
if not math.isfinite(value) or not self.lr_bounds[0]<=value<=self.lr_bounds[1]:raise FloatingPointError('Inner-LR guard')
def generate_chunk(adapter,prepared,noise,incoming):
"""The model computation receives no target, episode ID, split or source index."""
with torch.no_grad():prediction,candidate,trace=adapter.rollout(prepared,noise.clone(),incoming,differentiable=False)
if prediction.shape!=(noise.shape[0],10,32) or not torch.isfinite(prediction).all():raise ValueError('Prediction shape/finite')
if candidate:
if trace['commits']!=1 or trace['whole_vector_from_iteration']!=9:raise RuntimeError('Commit contract')
return prediction,candidate,trace
|