model/utils/training_mdn.py

# import copy
import numpy as np
from tqdm import tqdm
import torch
import gc
from torch.distributions import Normal
from loguru import logger
# from training import AverageMeter
# from mdn_utils import mdn_loss_fn
def mdn_loss_fn(pi, sigma, mu, y,dist_threhold=7.0,eps = 1e-10):
    mu = torch.clip(torch.nan_to_num(mu,0.0),min=1e-6)
    sigma = torch.clip(torch.nan_to_num(sigma,0.0),min=1e-6)
    pi = torch.clip(torch.nan_to_num(pi,0.0),min=1e-6)
    
    """calculate the mdn """
    normal = Normal(mu.real, sigma.real)
    loglik = normal.log_prob(y.expand_as(normal.loc))
    loss = -torch.logsumexp(torch.log(pi.real + eps) + loglik, dim=1)
    loss = loss[torch.where(y <= dist_threhold)[0]]
    loss = loss.mean()
    return torch.nan_to_num(loss,0.0)
# def mdn_loss_fn(pi, sigma, mu, y, eps=1e-10):
#     normal = Normal(mu, sigma)
#     #loss = th.exp(normal.log_prob(y.expand_as(normal.loc)))
#     #loss = th.sum(loss * pi, dim=1)
#     #loss = -th.log(loss)
#     loglik = normal.log_prob(y.expand_as(normal.loc))
#     loss = -th.logsumexp(th.log(pi + eps) + loglik, dim=1)
#     return loss
import torch as th

def mdn_loss_fn_min_diatance_atom(pi, sigma, mu, y,dist_threhold=7.0,eps = 1e-10,topN = 1):
    mu = torch.clip(torch.nan_to_num(mu,0.0),min=1e-6)
    sigma = torch.clip(torch.nan_to_num(sigma,0.0),min=1e-6)
    pi = torch.clip(torch.nan_to_num(pi,0.0),min=1e-6)

    """use ca- pose to calculate the mdn """
    normal = Normal(mu.real, sigma.real)
    loglik = normal.log_prob(y.expand_as(normal.loc))
    loss = -torch.logsumexp(torch.log(pi.real + eps) + loglik, dim=1)
    loss = loss[torch.where(y <= dist_threhold)[0]]
    loss = loss.mean()
    return loss
def calculate_probablity(pi, sigma, mu, y,dist_threhold=5.0,eps = 1e-10):
    mu = torch.clip(torch.nan_to_num(mu,0.0),min=1e-6)
    sigma = torch.clip(torch.nan_to_num(sigma,0.0),min=1e-6)
    pi = torch.clip(torch.nan_to_num(pi,0.0),min=1e-6)
    normal = Normal(mu.real, sigma.real)
    logprob = normal.log_prob(y.expand_as(normal.loc))
    logprob += torch.log(pi.real + eps )
    prob = logprob.exp().sum(1)
    prob[torch.where(y > dist_threhold)[0]] = 0.
    return prob

class AverageMeter():
    def __init__(self, types, unpooled_metrics=False, intervals=1):
        self.types = types
        self.intervals = intervals
        self.count = 0 if intervals == 1 else torch.zeros(len(types), intervals)
        self.acc = {t: torch.zeros(intervals) for t in types}
        self.unpooled_metrics = unpooled_metrics

    def add(self, vals, interval_idx=None):
        if self.intervals == 1:
            self.count += 1 if vals[0].dim() == 0 else len(vals[0])
            for type_idx, v in enumerate(vals):
                self.acc[self.types[type_idx]] += v.sum() if self.unpooled_metrics else v
        else:
            for type_idx, v in enumerate(vals):
                # logger.info(interval_idx[type_idx])
                # logger.info(v)
                # logger.info(interval_idx[type_idx], torch.ones(len(v)))
                self.count[type_idx].index_add_(0, interval_idx[type_idx], torch.ones(len(v)))
                if not torch.allclose(v, torch.tensor(0.0)):
                    self.acc[self.types[type_idx]].index_add_(0, interval_idx[type_idx], v)

    def summary(self):
        if self.intervals == 1:
            out = {k: v.item() / self.count for k, v in self.acc.items()}
            return out
        else:
            out = {}
            for i in range(self.intervals):
                for type_idx, k in enumerate(self.types):
                    out['int' + str(i) + '_' + k] = (
                            list(self.acc.values())[type_idx][i] / self.count[type_idx][i]).item()
            return out
def train_mdn_epoch(model, loader, optimizer, device,accelerator,ema_weights):
    model.train()
    meter = AverageMeter(['loss','mdn_loss_interaction', 'mdn_loss_ligand', 'atom_types_loss', 'bond_types_loss', 'residue_types_loss'],
                         unpooled_metrics=True)

    for data in loader:
    # for data in tqdm(loader, total=len(loader),disable=not accelerator.is_local_main_process):
        if device.type == 'cuda' and len(data) == 1 or device.type == 'cpu' and data.num_graphs == 1:
            logger.info("Skipping batch of size 1 since otherwise batchnorm would not work.")
        optimizer.zero_grad()
        try:
            # pi, sigma, mu, dist = model(data)
            with accelerator.autocast():
                mdn_loss_interaction , mdn_loss_ligand , atom_types_loss , bond_types_loss , residue_types_loss = model(data)#mdn_loss_fn(pi, sigma, mu, dist)
                # logger.info(loss)
                loss = mdn_loss_interaction + mdn_loss_ligand + 0.001*atom_types_loss + 0.001*bond_types_loss + 0.001*residue_types_loss
                accelerator.backward(loss)
                optimizer.step()
            # gather all loss for plot
            mdn_loss_interaction= accelerator.gather(mdn_loss_interaction)
            mdn_loss_ligand= accelerator.gather(mdn_loss_ligand)
            atom_types_loss= accelerator.gather(atom_types_loss)
            bond_types_loss= accelerator.gather(bond_types_loss)
            residue_types_loss= accelerator.gather(residue_types_loss)
            loss= accelerator.gather(loss)
            # logger.info('loss val: ',loss.mean().cpu().detach())
            metrics = [loss.mean().cpu().detach(),mdn_loss_interaction.mean().cpu().detach() , mdn_loss_ligand.mean().cpu().detach() , atom_types_loss.mean().cpu().detach() , bond_types_loss.mean().cpu().detach() , residue_types_loss.mean().cpu().detach()]
            meter.add(metrics)
            # logger.info('loss train: ',loss.mean().cpu().detach())
            ema_weights.update(model.parameters())
            # meter.add([loss.mean().cpu().detach()])
        except RuntimeError as e:
            if 'out of memory' in str(e):
                logger.info('| WARNING: ran out of memory, skipping batch')
                for p in model.parameters():
                    if p.grad is not None:
                        del p.grad  # free some memory
                optimizer.zero_grad()
                del data
                # loss = 0.0*sum([p.sum() for p in model.parameters() if p.requires_grad])
                # accelerator.backward(loss)
                # optimizer.step()
                gc.collect()
                torch.cuda.empty_cache()
                continue
            elif 'Input mismatch' in str(e):
                logger.info('| WARNING: weird torch_cluster error, skipping batch')
                for p in model.parameters():
                    if p.grad is not None:
                        del p.grad  # free some memory
                optimizer.zero_grad()
                del data
                gc.collect()
                torch.cuda.empty_cache()
                continue
            else:
                raise e
    return meter.summary()


def test_mdn_epoch(model, loader, device,accelerator, test_sigma_intervals=False):
    model.eval()
    meter = AverageMeter(['loss','mdn_loss_interaction', 'mdn_loss_ligand', 'atom_types_loss', 'bond_types_loss', 'residue_types_loss'],
                         unpooled_metrics=True)

    if test_sigma_intervals:
        meter_all = AverageMeter(
            ['loss'],
            unpooled_metrics=True, intervals=10)

    for data in loader:
        try:
            with torch.no_grad():
                    # pi, sigma, mu, dist,_ = model(data)
                with accelerator.autocast():
                    mdn_loss_interaction , mdn_loss_ligand , atom_types_loss , bond_types_loss , residue_types_loss,_ = model(data)#mdn_loss_fn(pi, sigma, mu, dist)
                    loss = mdn_loss_interaction + mdn_loss_ligand + 0.001*atom_types_loss + 0.001*bond_types_loss + 0.001*residue_types_loss
            mdn_loss_interaction= accelerator.gather(mdn_loss_interaction)
            mdn_loss_ligand= accelerator.gather(mdn_loss_ligand)
            atom_types_loss= accelerator.gather(atom_types_loss)
            bond_types_loss= accelerator.gather(bond_types_loss)
            residue_types_loss= accelerator.gather(residue_types_loss)
            loss= accelerator.gather(loss)
            # logger.info('loss val: ',loss.mean().cpu().detach())
            metrics = [loss.mean().cpu().detach(),mdn_loss_interaction.mean().cpu().detach() , mdn_loss_ligand.mean().cpu().detach() , atom_types_loss.mean().cpu().detach() , bond_types_loss.mean().cpu().detach() , residue_types_loss.mean().cpu().detach()]
            meter.add(metrics)

        except RuntimeError as e:
            if 'out of memory' in str(e):
                logger.info('| WARNING: ran out of memory, skipping batch')
                for p in model.parameters():
                    if p.grad is not None:
                        del p.grad  # free some memory
                del data
                gc.collect()
                torch.cuda.empty_cache()
                continue
            elif 'Input mismatch' in str(e):
                logger.info('| WARNING: weird torch_cluster error, skipping batch')
                for p in model.parameters():
                    if p.grad is not None:
                        del p.grad  # free some memory
                del data
                gc.collect()
                torch.cuda.empty_cache()
                continue
            else:
                raise e

    out = meter.summary()
    # if test_sigma_intervals > 0: out.update(meter_all.summary())
    return out