Name / remedi /utils.py
Username-IV
Created project
79df115
Raw History Blame Contribute Delete
3.55 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.parallel import DistributedDataParallel as DDP
try:
import deepspeed
from deepspeed import DeepSpeedEngine
except ImportError:
DeepSpeedEngine = None # Fallback in case DeepSpeed is not installed
def get_config(model: nn.Module):
"""
Retrieve the `config` attribute from a model, whether it's a plain nn.Module,
a DistributedDataParallel (DDP) model, or a DeepSpeed-wrapped model.
"""
# Plain nn.Module
if isinstance(model, nn.Module) and not isinstance(model, (DDP, DeepSpeedEngine if DeepSpeedEngine else tuple())):
if hasattr(model, "config"):
return model.config
# DDP-wrapped model
if isinstance(model, DDP):
if hasattr(model.module, "config"):
return model.module.config
# DeepSpeed-wrapped model
if DeepSpeedEngine and isinstance(model, DeepSpeedEngine):
# DeepSpeed wraps the original model in `module`, similar to DDP
if hasattr(model.module, "config"):
return model.module.config
raise AttributeError("The model (including its wrappers) does not have a `config` attribute.")
def reverse_cumsum(x, dim=-1):
return torch.flip(torch.cumsum(torch.flip(x, [dim]), dim=dim), [dim])
def sample_categorical(categorical_probs, method="hard"):
if method == "hard":
input_shape = categorical_probs.shape
N = input_shape[-1]
flat_probs = categorical_probs.reshape(-1, N)
samples = torch.multinomial(flat_probs, num_samples=1).squeeze(-1)
return samples.reshape(input_shape[:-1])
elif method == 'max':
return categorical_probs.argmax(dim=-1)
else:
raise ValueError(f"Method {method} for sampling categorical variables is not valid.")
def get_pt_logprobs(remask_prob, sample_order):
batch_size = remask_prob.shape[0]
bs_idx = torch.arange(batch_size, dtype=sample_order.dtype).unsqueeze(1)
sample_flag = torch.zeros_like(remask_prob, dtype=torch.bool)
sample_flag[bs_idx, sample_order] = True
# F-set sum
sum_neg = (remask_prob * (~sample_flag)).sum(-1, keepdim=True) # bs, 1
# (bs, gen_length // steps)
select_remask = remask_prob[bs_idx, sample_order]
# (bs, 1) + (bs, gen_length // steps)
fen_mu_log = (sum_neg + reverse_cumsum(select_remask, dim=-1)).log()
pt_logprobs = select_remask.log() - fen_mu_log
return pt_logprobs
def interpolate_p(prob, x0, mask_id=126336, alpha=0):
"""
Interpolate between the probability distribution `prob` and a one-hot distribution
based on whether each token in `x0` is equal to `mask_id`.
Args:
prob (FloatTensor): shape (bs, len, vocab) — predicted probability distributions.
x0 (LongTensor): shape (bs, len) — token ids.
mask_id (int): ID to compare in x0 to determine interpolation behavior.
alpha (float): interpolation weight (0 = only one-hot, 1 = only prob).
Returns:
output_prob (FloatTensor): interpolated output distribution, shape (bs, len, vocab).
"""
bs, length, vocab = prob.shape
# Create one-hot from x0
one_hot = F.one_hot(x0, num_classes=vocab).float()
# Condition for where to apply interpolation
mask = (x0 != mask_id).unsqueeze(-1) # shape (bs, len, 1)
# Interpolation
output_prob = torch.where(
mask, # where x0 != mask_id
alpha * prob + (1 - alpha) * one_hot,
prob # where x0 == mask_id
)
return output_prob