Download t6/loss.py from auralray/AMPRecognitionBenchmark: direct link, hf CLI and curl.
- Browser
- Download file 15.7 kB
-
https://huggingface.co/auralray/AMPRecognitionBenchmark/resolve/main/t6/loss.py
- Command line
-
hf download hf://auralray/AMPRecognitionBenchmark/t6/loss.py
-
curl -L -o loss.py https://huggingface.co/auralray/AMPRecognitionBenchmark/resolve/main/t6/loss.py
15.7 kB
| import torch | |
| from torch import nn | |
| import torch.nn.functional as F | |
| from math import cos, pi, sin | |
| import math | |
| import numpy as np | |
| from scipy.special import lambertw | |
| def mixup_criterion(criterion, pred, y_a, y_b, lam, pow=2): | |
| y = lam ** pow * y_a + (1 - lam) ** pow * y_b | |
| return criterion(pred, y) | |
| def mixup_data(v, q, a): | |
| '''Returns mixed inputs, pairs of targets, and lambda without organ constraint''' | |
| lam = np.random.beta(1, 1) | |
| batch_size = v.shape[0] | |
| index = torch.randperm(batch_size) | |
| mixed_v = lam * v + (1 - lam) * v[index, :] | |
| mixed_q = lam * q + (1 - lam) * q[index, :] | |
| a_1, a_2 = a, a[index] | |
| return mixed_v, mixed_q, a_1, a_2, lam | |
| def linear(epoch, nepoch): | |
| return 1 - epoch / nepoch | |
| def convex(epoch, nepoch): | |
| return epoch / (2 - nepoch) | |
| def concave(epoch, nepoch): | |
| return 1 - sin((epoch / nepoch) * (pi / 2)) | |
| def composite(epoch, nepoch): | |
| return 0.5 * cos((epoch / nepoch) * pi) + 0.5 | |
| class LogCoshLoss(nn.Module): | |
| def __init__(self): | |
| super().__init__() | |
| def forward(self, y_t, y_prime_t): | |
| ey_t = y_t - y_prime_t | |
| return torch.mean(torch.log(torch.cosh(ey_t + 1e-12)))+F.mse_loss(y_t, y_prime_t) | |
| class MLCE(nn.Module): | |
| def __init__(self): | |
| super(MLCE, self).__init__() | |
| def _mlcce(self, y_pred, y_true): | |
| y_pred = (1 - 2 * y_true) * y_pred | |
| y_pred_neg = y_pred - y_true * 1e12 | |
| y_pred_pos = y_pred - (1 - y_true) * 1e12 | |
| zeros = torch.zeros_like(y_pred[..., :1]) | |
| y_pred_neg = torch.cat([y_pred_neg, zeros], dim=-1) | |
| y_pred_pos = torch.cat([y_pred_pos, zeros], dim=-1) | |
| neg_loss = torch.logsumexp(y_pred_neg, dim=-1) | |
| pos_loss = torch.logsumexp(y_pred_pos, dim=-1) | |
| loss = torch.mean(neg_loss + pos_loss) | |
| return loss | |
| def __call__(self, y_pred, y_true): | |
| return self._mlcce(y_pred, y_true) | |
| class SuperLoss(nn.Module): | |
| def __init__(self, C=10, lam=1, batch_size=128): | |
| super(SuperLoss, self).__init__() | |
| self.tau = math.log(C) | |
| self.lam = lam # set to 1 for CIFAR10 and 0.25 for CIFAR100 | |
| self.batch_size = batch_size | |
| def forward(self, logits, targets): | |
| l_i = F.mse_loss(logits, targets, reduction='none').detach() | |
| sigma = self.sigma(l_i) | |
| loss = (F.mse_loss(logits, targets, reduction='none') - self.tau) * sigma + self.lam * ( | |
| torch.log(sigma) ** 2) | |
| loss = loss.sum() / self.batch_size | |
| return loss | |
| def sigma(self, l_i): | |
| x = torch.ones(l_i.size()) * (-2 / math.exp(1.)) | |
| x = x.cuda() | |
| y = 0.5 * torch.max(x, (l_i - self.tau) / self.lam) | |
| y = y.cpu().numpy() | |
| sigma = np.exp(-lambertw(y)) | |
| sigma = sigma.real.astype(np.float32) | |
| sigma = torch.from_numpy(sigma).cuda() | |
| return sigma | |
| def unbiased_curriculum_loss(out, data, args, epoch, epochs, scheduler='linear'): | |
| losses = [] | |
| scheduler = linear if scheduler == 'linear' else concave | |
| # calculate difficulty measurement function | |
| adjusted_losses = [] | |
| for idx in range(out.shape[0]): | |
| ground_truth = max(1, abs(data[idx].item())) | |
| loss = F.mse_loss(out[idx], data[idx]) | |
| losses.append(loss) | |
| adjusted_losses.append(loss.item() / ground_truth) | |
| mean_loss, std_loss = np.mean(adjusted_losses), np.std(adjusted_losses) | |
| # re-weight losses | |
| total_loss = 0 | |
| for i, loss in enumerate(losses): | |
| if adjusted_losses[i] > mean_loss + 1 * std_loss: | |
| schedule_factor = scheduler(epoch, args.epochs) | |
| total_loss += schedule_factor * loss | |
| else: | |
| total_loss += loss | |
| return total_loss | |
| def reduce_loss(loss, reduction): | |
| """Reduce loss as specified. | |
| Args: | |
| loss (Tensor): Elementwise loss tensor. | |
| reduction (str): Options are "none", "mean" and "sum". | |
| Return: | |
| Tensor: Reduced loss tensor. | |
| """ | |
| # none: 0, elementwise_mean:1, sum: 2 | |
| if reduction == 'mean': | |
| return loss.mean() | |
| elif reduction == 'sum': | |
| return loss.sum() | |
| else: | |
| return loss | |
| def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): | |
| """Apply element-wise weight and reduce loss. | |
| Args: | |
| loss (Tensor): Element-wise loss. | |
| weight (Tensor): Element-wise weights. | |
| reduction (str): Same as built-in losses of PyTorch. | |
| avg_factor (float): Avarage factor when computing the mean of losses. | |
| Returns: | |
| Tensor: Processed loss values. | |
| """ | |
| # if weight is specified, apply element-wise weight | |
| if weight is not None: | |
| loss = loss * weight | |
| # if avg_factor is not specified, just reduce the loss | |
| if avg_factor is None: | |
| loss = reduce_loss(loss, reduction) | |
| else: | |
| # if reduction is mean, then average the loss by avg_factor | |
| if reduction == 'mean': | |
| loss = loss.sum() / avg_factor | |
| # if reduction is 'none', then do nothing, otherwise raise an error | |
| elif reduction != 'none': | |
| raise ValueError('avg_factor can not be used with reduction="sum"') | |
| return loss | |
| def _squeeze_binary_labels(label): | |
| if label.size(1) == 1: | |
| squeeze_label = label.view(len(label), -1) | |
| else: | |
| inds = torch.nonzero(label >= 1).squeeze() | |
| squeeze_label = inds[:,-1] | |
| return squeeze_label | |
| def cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None): | |
| # element-wise losses | |
| if label.size(-1) != pred.size(0): | |
| label = _squeeze_binary_labels(label) | |
| loss = F.cross_entropy(pred, label, reduction='none') | |
| # apply weights and do the reduction | |
| if weight is not None: | |
| weight = weight.float() | |
| loss = weight_reduce_loss( | |
| loss, weight=weight, reduction=reduction, avg_factor=avg_factor) | |
| return loss | |
| def _expand_binary_labels(labels, label_weights, label_channels): | |
| bin_labels = labels.new_full((labels.size(0), label_channels), 0) | |
| inds = torch.nonzero(labels >= 1).squeeze() | |
| if inds.numel() > 0: | |
| bin_labels[inds, labels[inds] - 1] = 1 | |
| if label_weights is None: | |
| bin_label_weights = None | |
| else: | |
| bin_label_weights = label_weights.view(-1, 1).expand( | |
| label_weights.size(0), label_channels) | |
| return bin_labels, bin_label_weights | |
| def binary_cross_entropy(pred, | |
| label, | |
| weight=None, | |
| reduction='mean', | |
| avg_factor=None): | |
| if pred.dim() != label.dim(): | |
| label, weight = _expand_binary_labels(label, weight, pred.size(-1)) | |
| # weighted element-wise losses | |
| if weight is not None: | |
| weight = weight.float() | |
| loss = F.binary_cross_entropy_with_logits( | |
| pred, label.float(), weight, reduction='none') | |
| loss = weight_reduce_loss(loss, reduction=reduction, avg_factor=avg_factor) | |
| return loss | |
| def partial_cross_entropy(pred, | |
| label, | |
| weight=None, | |
| reduction='mean', | |
| avg_factor=None): | |
| if pred.dim() != label.dim(): | |
| label, weight = _expand_binary_labels(label, weight, pred.size(-1)) | |
| # weighted element-wise losses | |
| if weight is not None: | |
| weight = weight.float() | |
| mask = label == -1 | |
| loss = F.binary_cross_entropy_with_logits( | |
| pred, label.float(), weight, reduction='none') | |
| if mask.sum() > 0: | |
| loss *= (1-mask).float() | |
| avg_factor = (1-mask).float().sum() | |
| # do the reduction for the weighted loss | |
| loss = weight_reduce_loss(loss, reduction=reduction, avg_factor=avg_factor) | |
| return loss | |
| class ResampleLoss(nn.Module): | |
| def __init__(self, | |
| class_freq: torch.Tensor, | |
| neg_class_freq: torch.Tensor, | |
| use_sigmoid=True, | |
| reduction='mean', | |
| loss_weight=1.0, | |
| partial=False, | |
| focal=dict( | |
| focal=True, | |
| balance_param=2.0, | |
| gamma=2, | |
| ), | |
| CB_loss=dict( | |
| CB_beta=0.9, | |
| CB_mode='average_w' # 'by_class', 'average_n', 'average_w', 'min_n' | |
| ), | |
| map_param=dict( | |
| alpha=10.0, | |
| beta=0.2, | |
| gamma=0.1 | |
| ), | |
| logit_reg=dict( | |
| neg_scale=5.0, | |
| init_bias=0.1 | |
| ), | |
| reweight_func=None, # None, 'inv', 'sqrt_inv', 'rebalance', 'CB' | |
| weight_norm=None # None, 'by_instance', 'by_batch' | |
| ): | |
| super(ResampleLoss, self).__init__() | |
| assert (use_sigmoid is True) or (partial is False) | |
| assert class_freq.device == neg_class_freq.device | |
| self.use_sigmoid = use_sigmoid | |
| self.partial = partial | |
| self.loss_weight = loss_weight | |
| self.reduction = reduction | |
| self.device = class_freq.device | |
| if self.use_sigmoid: | |
| if self.partial: | |
| self.cls_criterion = partial_cross_entropy | |
| else: | |
| self.cls_criterion = binary_cross_entropy | |
| else: | |
| self.cls_criterion = cross_entropy | |
| # reweighting function | |
| self.reweight_func = reweight_func | |
| # normalization (optional) | |
| self.weight_norm = weight_norm | |
| # focal loss params | |
| self.focal = focal['focal'] | |
| self.gamma = focal['gamma'] | |
| self.balance_param = focal['balance_param'] | |
| # mapping function params | |
| self.map_alpha = map_param['alpha'] | |
| self.map_beta = map_param['beta'] | |
| self.map_gamma = map_param['gamma'] | |
| # CB loss params (optional) | |
| self.CB_beta = CB_loss['CB_beta'] | |
| self.CB_mode = CB_loss['CB_mode'] | |
| self.class_freq = class_freq.float() | |
| self.neg_class_freq = neg_class_freq.float() | |
| self.num_classes = self.class_freq.shape[0] | |
| self.train_num = self.class_freq[0] + self.neg_class_freq[0] | |
| # regularization params | |
| self.logit_reg = logit_reg | |
| self.neg_scale = logit_reg[ | |
| 'neg_scale'] if 'neg_scale' in logit_reg else 1.0 | |
| init_bias = logit_reg['init_bias'] if 'init_bias' in logit_reg else 0.0 | |
| self.init_bias = - torch.log( | |
| self.train_num / self.class_freq - 1) * init_bias / self.neg_scale | |
| self.freq_inv = torch.ones(self.class_freq.shape).to(self.device) / self.class_freq | |
| self.propotion_inv = self.train_num / self.class_freq | |
| # print('\033[1;35m loading from {} | {} | {} | s\033[0;0m'.format(freq_file, reweight_func, logit_reg)) | |
| # print('\033[1;35m rebalance reweighting mapping params: {:.2f} | {:.2f} | {:.2f} \033[0;0m'.format(self.map_alpha, self.map_beta, self.map_gamma)) | |
| def forward(self, | |
| cls_score, | |
| label, | |
| avg_factor=None, | |
| **kwargs): | |
| weight = self.reweight_functions(label) | |
| cls_score, weight = self.logit_reg_functions(label.float(), cls_score, weight) | |
| if self.focal: | |
| logpt = - self.cls_criterion( | |
| cls_score.clone(), label, weight=None, reduction='none', | |
| avg_factor=avg_factor) | |
| # pt is sigmoid(logit) for pos or sigmoid(-logit) for neg | |
| pt = torch.exp(logpt) | |
| loss = self.cls_criterion( | |
| cls_score, label.float(), weight=weight, reduction='none') | |
| loss = ((1 - pt) ** self.gamma) * loss | |
| loss = self.balance_param * loss | |
| loss = reduce_loss(loss, reduction=self.reduction) | |
| else: | |
| loss = self.cls_criterion(cls_score, label.float(), weight, | |
| reduction=self.reduction) | |
| loss = self.loss_weight * loss | |
| return loss | |
| def reweight_functions(self, label): | |
| if self.reweight_func is None: | |
| return None | |
| elif self.reweight_func in ['inv', 'sqrt_inv']: | |
| weight = self.RW_weight(label.float()) | |
| elif self.reweight_func in 'rebalance': | |
| weight = self.rebalance_weight(label.float()) | |
| elif self.reweight_func in 'CB': | |
| weight = self.CB_weight(label.float()) | |
| else: | |
| return None | |
| if self.weight_norm is not None: | |
| if 'by_instance' in self.weight_norm: | |
| max_by_instance, _ = torch.max(weight, dim=-1, keepdim=True) | |
| weight = weight / max_by_instance | |
| elif 'by_batch' in self.weight_norm: | |
| weight = weight / torch.max(weight) | |
| return weight | |
| def logit_reg_functions(self, labels, logits, weight=None): | |
| if not self.logit_reg: | |
| return logits, weight | |
| if 'init_bias' in self.logit_reg: | |
| logits += self.init_bias | |
| if 'neg_scale' in self.logit_reg: | |
| logits = logits * (1 - labels) * self.neg_scale + logits * labels | |
| weight = weight / self.neg_scale * (1 - labels) + weight * labels | |
| return logits, weight | |
| def rebalance_weight(self, gt_labels): | |
| repeat_rate = torch.sum( gt_labels.float() * self.freq_inv, dim=1, keepdim=True) | |
| pos_weight = self.freq_inv.clone().detach().unsqueeze(0) / repeat_rate | |
| # pos and neg are equally treated | |
| weight = torch.sigmoid(self.map_beta * (pos_weight - self.map_gamma)) + self.map_alpha | |
| return weight | |
| def CB_weight(self, gt_labels): | |
| if 'by_class' in self.CB_mode: | |
| weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ | |
| (1 - torch.pow(self.CB_beta, self.class_freq)).to(self.device) | |
| elif 'average_n' in self.CB_mode: | |
| avg_n = torch.sum(gt_labels * self.class_freq, dim=1, keepdim=True) / \ | |
| torch.sum(gt_labels, dim=1, keepdim=True) | |
| weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ | |
| (1 - torch.pow(self.CB_beta, avg_n)).to(self.device) | |
| elif 'average_w' in self.CB_mode: | |
| weight_ = torch.tensor((1 - self.CB_beta)).to(self.device) / \ | |
| (1 - torch.pow(self.CB_beta, self.class_freq)).to(self.device) | |
| weight = torch.sum(gt_labels * weight_, dim=1, keepdim=True) / \ | |
| torch.sum(gt_labels, dim=1, keepdim=True) | |
| elif 'min_n' in self.CB_mode: | |
| min_n, _ = torch.min(gt_labels * self.class_freq + | |
| (1 - gt_labels) * 100000, dim=1, keepdim=True) | |
| weight = torch.tensor((1 - self.CB_beta)).to(self.device) / \ | |
| (1 - torch.pow(self.CB_beta, min_n)).to(self.device) | |
| else: | |
| raise NameError | |
| return weight | |
| def RW_weight(self, gt_labels, by_class=True): | |
| if 'sqrt' in self.reweight_func: | |
| weight = torch.sqrt(self.propotion_inv) | |
| else: | |
| weight = self.propotion_inv | |
| if not by_class: | |
| sum_ = torch.sum(weight * gt_labels, dim=1, keepdim=True) | |
| weight = sum_ / torch.sum(gt_labels, dim=1, keepdim=True) | |
| return weight |