ReflectionEraserApp / DSRNet /util /schedulers.py
SunlitDeadline's picture
commited from namirahrasul/DSRNet -main local
b9e72b4 verified
Raw History Blame Contribute Delete
4.6 kB
from torch.optim.lr_scheduler import _LRScheduler
#Learning rate scheduler
from torch.optim.lr_scheduler import ReduceLROnPlateau
#Reduce learning rate when a metric has stopped improving. Models often benefit from reducing the learning rate by a factor of 2-10 once learning stagnates
class GradualWarmupScheduler(_LRScheduler):
#Starts with a small learning rate and gradually increases it over a few initial epochs or iterations. This is particularly useful in preventing the model from diverging in the initial phase of training and is often used in training deep networks from scratch.
#Pros: Prevents early divergence; stabilizes training.
#Cons: Requires tuning of warm-up duration and rate limits.
#Use Case: Crucial for training deep networks from scratch.
"""(mingcv) Gradually warm-up(increasing) learning rate in optimizer.
Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'.
Args:
optimizer (Optimizer): Wrapped optimizer.
multiplier: target learning rate = base lr * multiplier if multiplier > 1.0. if multiplier = 1.0, lr starts from 0 and ends up with the base_lr.
total_epoch: target learning rate is reached at total_epoch, gradually
after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau)
"""
#total_epoch (number of epochs to reach the target learning rate
#after scheduler: scheduler after warmup
def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):
self.multiplier = multiplier
if self.multiplier < 1.:
raise ValueError('multiplier should be greater thant or equal to 1.')
self.total_epoch = total_epoch
self.after_scheduler = after_scheduler
self.finished = False #initializes a flag to indicate if warm-up is finished.
super(GradualWarmupScheduler, self).__init__(optimizer)
def get_lr(self): #get the current learning rates.
if self.last_epoch > self.total_epoch: #warm is ovcer , target lr reached
if self.after_scheduler: #plateuLR used
if not self.finished: #sets the base learning rates for the after_scheduler to be multiplier times the base learning rates and returns rlearningrate of after scheduler.
self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] #self.base_lrs intialized from _LRScheduler
self.finished = True #warm up done
return self.after_scheduler.get_lr()
return [base_lr * self.multiplier for base_lr in self.base_lrs]
if self.multiplier == 1.0: #increase lienarly and propertionally to number of epochs run
return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]
else: #increaase lr non linearly
return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
self.base_lrs]
"??? why this formula for non linear"
def step_ReduceLROnPlateau(self, metrics, epoch=None):
#metrics: A value that will be monitored (e.g., validation loss) to decide if the learning rate needs adjustment.
if epoch is None: #not resuming from a specific epoch
epoch = self.last_epoch + 1
self.last_epoch = epoch if epoch != 0 else 1 # (mingcv) ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning
if self.last_epoch <= self.total_epoch:
warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in
self.base_lrs]
for param_group, lr in zip(self.optimizer.param_groups, warmup_lr):
param_group['lr'] = lr
else:
if epoch is None:
self.after_scheduler.step(metrics, None)
else:
self.after_scheduler.step(metrics, epoch - self.total_epoch)
def step(self, epoch=None, metrics=None):
if type(self.after_scheduler) != ReduceLROnPlateau:
if self.finished and self.after_scheduler:
if epoch is None:
self.after_scheduler.step(None)
else:
self.after_scheduler.step(epoch - self.total_epoch)
else:
return super(GradualWarmupScheduler, self).step(epoch)
else:
self.step_ReduceLROnPlateau(metrics, epoch)