manbeast3b commited on
Commit
d38ea92
·
verified ·
1 Parent(s): 951bac8

Update src/loss.py

Browse files
Files changed (1) hide show
  1. src/loss.py +59 -17
src/loss.py CHANGED
@@ -3,8 +3,6 @@ import torch
3
  from tqdm import tqdm
4
  import numpy as np
5
 
6
-
7
-
8
  class DynamicBandMatrix(torch.nn.Module):
9
  def __init__(self, size, band_width):
10
  super().__init__()
@@ -54,24 +52,49 @@ class LearnableScaler(torch.nn.Module):
54
  return self.scale_factors * torch.log(1 + torch.tensor(timestep)).to('cuda')
55
 
56
  class HybridMatrix(torch.nn.Module):
57
- def __init__(self, size):
58
- super().__init__()
59
- self.exp_matrix = ExponentialScalingMatrix(size)
60
- self.sin_matrix = SinusoidalMatrix(size)
 
 
 
 
 
 
 
 
 
 
 
61
 
62
- def forward(self, timestep):
63
- exp_part = self.exp_matrix(timestep)
64
- sin_part = self.sin_matrix(timestep)
65
- return exp_part + sin_part
66
-
 
 
 
 
 
 
 
 
 
 
 
 
67
  class LossSchedulerModel(torch.nn.Module):
68
- def __init__(A,wx,we, strategy="HybridMatrix"):
69
  super(LossSchedulerModel,A).__init__()
70
  assert len(wx.shape)==1 and len(we.shape)==2
71
  B=wx.shape[0];assert B==we.shape[0]and B==we.shape[1]
72
  A.register_parameter('wx',torch.nn.Parameter(wx))
73
  A.register_parameter('we',torch.nn.Parameter(we))
74
  size=13
 
 
75
  if strategy == "dynamic_band":
76
  A.matrix_generator = DynamicBandMatrix(size, band_width=3)
77
  elif strategy == "sinusoidal":
@@ -86,12 +109,25 @@ class LossSchedulerModel(torch.nn.Module):
86
  A.matrix_generator = LearnableScaler(size)
87
  else:
88
  raise ValueError("Unknown strategy")
 
89
  def forward(A,t,xT,e_prev):
90
- B=e_prev;assert t-len(B)+1==0;C=xT*A.wx[t];jitter=A.matrix_generator.forward(t)
91
- for(D,E,J)in zip(B,A.we[t],jitter[t]):C+=D*(E+J.to(D.device))
92
- return C.to(xT.dtype)
 
 
 
 
 
 
93
  class LossScheduler:
94
- def __init__(A,timesteps,model):A.timesteps=timesteps;A.model=model;A.init_noise_sigma=1.;A.order=1
 
 
 
 
 
 
95
  @staticmethod
96
  def load(path):A,B,C=torch.load(path,map_location='cpu');D=LossSchedulerModel(B,C);return LossScheduler(A,D)
97
  def save(A,path):B,C,D=A.timesteps,A.model.wx,A.model.we;torch.save((B,C,D),path)
@@ -101,7 +137,13 @@ class LossScheduler:
101
  def step(self,model_output,timestep,sample,*D,**E):
102
  A=self;B=A.timesteps.tolist().index(timestep);assert A.t_prev==-1 or B==A.t_prev+1
103
  if A.t_prev==-1:A.xT=sample
104
- A.e_prev.append(model_output);C=A.model(B,A.xT,A.e_prev)
 
 
 
 
 
 
105
  if B+1==len(A.timesteps):A.xT=_A;A.e_prev=[];A.t_prev=-1
106
  else:A.t_prev=B
107
  return C,
 
3
  from tqdm import tqdm
4
  import numpy as np
5
 
 
 
6
  class DynamicBandMatrix(torch.nn.Module):
7
  def __init__(self, size, band_width):
8
  super().__init__()
 
52
  return self.scale_factors * torch.log(1 + torch.tensor(timestep)).to('cuda')
53
 
54
  class HybridMatrix(torch.nn.Module):
55
+ def __init__(self, size):
56
+ super().__init__()
57
+ self.exp_matrix = ExponentialScalingMatrix(size)
58
+ self.sin_matrix = SinusoidalMatrix(size)
59
+
60
+ def forward(self, timestep):
61
+ exp_part = self.exp_matrix(timestep)
62
+ sin_part = self.sin_matrix(timestep)
63
+ return exp_part + sin_part
64
+
65
+ class RecurrentDynamicScaling(nn.Module):
66
+ def __init__(self, size):
67
+ super().__init__()
68
+ self.rnn = nn.GRUCell(size, size)
69
+ self.state = nn.Parameter(torch.zeros(size)) # Learnable state
70
 
71
+ def forward(self, timestep, x, correction):
72
+ self.state = self.rnn(self.state, correction) # Gradients flow through GRU
73
+ scaled_correction = correction * torch.sigmoid(self.state) # Scale dynamically
74
+ return x + scaled_correction
75
+
76
+ class AttentionDynamicScaling(nn.Module):
77
+ def __init__(self, size):
78
+ super().__init__()
79
+ self.attention = nn.MultiheadAttention(size, num_heads=1)
80
+
81
+ def forward(self, timestep, x, correction):
82
+ query = correction.unsqueeze(0) # Shape (1, batch, size)
83
+ key = x.unsqueeze(0)
84
+ value = correction.unsqueeze(0)
85
+ scaled_correction, _ = self.attention(query, key, value)
86
+ return x + scaled_correction.squeeze(0)
87
+
88
  class LossSchedulerModel(torch.nn.Module):
89
+ def __init__(A,wx,we, strategy="exponential_scaling"):
90
  super(LossSchedulerModel,A).__init__()
91
  assert len(wx.shape)==1 and len(we.shape)==2
92
  B=wx.shape[0];assert B==we.shape[0]and B==we.shape[1]
93
  A.register_parameter('wx',torch.nn.Parameter(wx))
94
  A.register_parameter('we',torch.nn.Parameter(we))
95
  size=13
96
+ A.scale_factor = nn.Parameter(torch.ones(size))
97
+ A.decay_rate = 0.1
98
  if strategy == "dynamic_band":
99
  A.matrix_generator = DynamicBandMatrix(size, band_width=3)
100
  elif strategy == "sinusoidal":
 
109
  A.matrix_generator = LearnableScaler(size)
110
  else:
111
  raise ValueError("Unknown strategy")
112
+
113
  def forward(A,t,xT,e_prev):
114
+ B=e_prev
115
+ assert t-len(B)+1==0;C=xT*A.wx[t]
116
+ jitter=A.matrix_generator.forward(t)
117
+ for(D,E,J)in zip(B,A.we[t],jitter[t]):
118
+ dynamic_scale = torch.tanh(A.decay_rate * t)
119
+ C+=D*(E+J.to(D.device))
120
+ scaled_xT += C * dynamic_scale
121
+ return scaled_xT.to(xT.dtype) #C.to(xT.dtype)
122
+
123
  class LossScheduler:
124
+ def __init__(A,timesteps,model, lr=0.01):
125
+ A.timesteps=timesteps
126
+ A.model=model
127
+ A.init_noise_sigma=1.
128
+ A.order=1
129
+ A.optimizer = torch.optim.SGD([A.model.scale_factor], lr=lr)
130
+
131
  @staticmethod
132
  def load(path):A,B,C=torch.load(path,map_location='cpu');D=LossSchedulerModel(B,C);return LossScheduler(A,D)
133
  def save(A,path):B,C,D=A.timesteps,A.model.wx,A.model.we;torch.save((B,C,D),path)
 
137
  def step(self,model_output,timestep,sample,*D,**E):
138
  A=self;B=A.timesteps.tolist().index(timestep);assert A.t_prev==-1 or B==A.t_prev+1
139
  if A.t_prev==-1:A.xT=sample
140
+ A.e_prev.append(model_output)
141
+ with torch.enable_grad():
142
+ A.optimizer.zero_grad()
143
+ C=A.model(B,A.xT,A.e_prev)
144
+ L=torch.nn.functional.mse_loss(C, model_output)
145
+ L.backward()
146
+ A.optimizer.step()
147
  if B+1==len(A.timesteps):A.xT=_A;A.e_prev=[];A.t_prev=-1
148
  else:A.t_prev=B
149
  return C,