fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 4)
0d48417 verified
Raw History Blame Contribute Delete
2.28 kB
"""Masked losses for multi-output regression and distribution heads."""
from __future__ import annotations
from typing import Any, Dict
import torch
import torch.nn.functional as F
LOSS_REGISTRY = {}
def register_loss(name: str):
def deco(fn):
LOSS_REGISTRY[name] = fn
return fn
return deco
def _reduce(loss: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
mask = mask.to(loss.dtype)
denom = mask.sum().clamp_min(1.0)
return (loss * mask).sum() / denom
@register_loss("masked_mae")
def masked_mae(pred, target, mask, **kwargs):
loss = (pred - target).abs()
return _reduce(loss, mask)
@register_loss("masked_mse")
def masked_mse(pred, target, mask, **kwargs):
loss = (pred - target) ** 2
return _reduce(loss, mask)
@register_loss("masked_huber")
def masked_huber(pred, target, mask, beta: float = 1.0, **kwargs):
loss = F.smooth_l1_loss(pred, target, reduction="none", beta=beta)
return _reduce(loss, mask)
@register_loss("masked_smape")
def masked_smape(pred, target, mask, eps: float = 1e-6, **kwargs):
denom = (target.abs() + pred.abs()) / 2.0 + 1.0
loss = (pred - target).abs() / denom.clamp_min(eps)
return _reduce(loss, mask)
@register_loss("masked_cross_entropy")
def masked_cross_entropy(pred, target, mask, label_smoothing: float = 0.0, **kwargs):
# pred: (B, T, C), target: (B, T), mask: (B, T)
if pred.dim() != 3:
raise ValueError(f"masked_cross_entropy expects (B,T,C), got {tuple(pred.shape)}")
C = pred.shape[-1]
logits = pred.reshape(-1, C)
labels = target.reshape(-1).long().clamp(0, C - 1)
m = mask.reshape(-1).to(logits.dtype)
loss = F.cross_entropy(logits, labels, reduction="none", label_smoothing=label_smoothing)
denom = m.sum().clamp_min(1.0)
return (loss * m).sum() / denom
def build_loss(cfg: Dict[str, Any]):
name = cfg["name"]
if name not in LOSS_REGISTRY:
raise KeyError(f"Unknown loss '{name}'. Available: {sorted(LOSS_REGISTRY)}")
params = dict(cfg.get("params", {}))
fn = LOSS_REGISTRY[name]
def loss_fn(pred, target, mask):
return fn(pred, target, mask, **params)
loss_fn.loss_name = name
return loss_fn
def list_losses():
return sorted(LOSS_REGISTRY)