Download code/minijev/training/losses.py from markuz89/minijev-general: direct link, hf CLI and curl.
- Browser
- Download file 3.18 kB
-
https://huggingface.co/markuz89/minijev-general/resolve/main/code/minijev/training/losses.py
- Command line
-
hf download hf://markuz89/minijev-general/code/minijev/training/losses.py
-
curl -L -o losses.py https://huggingface.co/markuz89/minijev-general/resolve/main/code/minijev/training/losses.py
3.18 kB
| """Decision losses over a variable number of choices (absent choices are masked). | |
| hard cross-entropy vs. the target index (optional label smoothing over *present* choices) | |
| soft cross-entropy vs. a soft label distribution | |
| kd Hinton distillation: T^2 * KL(teacher_T || student_T) from teacher logits | |
| score BCE of the scalar score head vs a continuous target in [0, 1] | |
| `distillation: {enabled: true, alpha: 0.7, temperature: 2.0}` is shorthand for | |
| hard_weight = 1 - alpha, kd_weight = alpha, kd_temperature = temperature. | |
| Any combination can be active; rows without a given signal are ignored. | |
| """ | |
| from __future__ import annotations | |
| import torch | |
| import torch.nn.functional as F | |
| def resolve_loss_cfg(cfg: dict | None) -> dict: | |
| """`distillation: {enabled, alpha, temperature}` -> hard weight (1 - alpha), KD weight alpha, KD T.""" | |
| cfg = dict(cfg or {}) | |
| dist = cfg.get("distillation") or {} | |
| if dist.get("enabled"): | |
| a = float(dist.get("alpha", 0.5)) | |
| cfg.update(hard_weight=1.0 - a, kd_weight=a, kd_temperature=float(dist.get("temperature", 2.0))) | |
| return cfg | |
| def decision_loss(logits: torch.Tensor, choice_mask: torch.Tensor, batch: dict, cfg: dict | None = None, | |
| score_logit: torch.Tensor | None = None): | |
| cfg = resolve_loss_cfg(cfg) | |
| logp = F.log_softmax(logits, -1) | |
| terms: dict[str, torch.Tensor] = {} | |
| total = logits.new_zeros(()) | |
| w_hard = cfg.get("hard_weight", 1.0) | |
| if w_hard and "target" in batch: | |
| target = batch["target"] | |
| valid = target >= 0 | |
| if valid.any(): | |
| eps = cfg.get("label_smoothing", 0.0) | |
| tgt = F.one_hot(target.clamp(min=0), logits.shape[-1]).float() | |
| if eps > 0: | |
| m = choice_mask.float() | |
| tgt = (1 - eps) * tgt + eps * m / m.sum(-1, keepdim=True) | |
| ce = -(tgt * logp.masked_fill(~choice_mask, 0.0)).sum(-1) | |
| terms["hard"] = ce[valid].mean() | |
| total = total + w_hard * terms["hard"] | |
| w_soft = cfg.get("soft_weight", 1.0) | |
| if w_soft and "soft" in batch: | |
| soft = batch["soft"] | |
| terms["soft"] = -(soft * logp.masked_fill(~choice_mask, 0.0)).sum(-1).mean() | |
| total = total + w_soft * terms["soft"] | |
| w_kd = cfg.get("kd_weight", 1.0) | |
| if w_kd and "teacher_logits" in batch: | |
| T = cfg.get("kd_temperature", 2.0) | |
| t = F.log_softmax(batch["teacher_logits"] / T, -1) | |
| s = F.log_softmax(logits / T, -1) | |
| kl = (t.exp() * (t - s)).masked_fill(~choice_mask, 0.0).sum(-1) | |
| has = batch["teacher_logits"].max(-1).values > -1e8 # rows without a teacher are all -1e9 | |
| if has.any(): | |
| terms["kd"] = kl[has].mean() * T * T | |
| total = total + w_kd * terms["kd"] | |
| w_score = cfg.get("score_weight", 1.0) | |
| if w_score and score_logit is not None and "score_target" in batch: | |
| y = batch["score_target"] | |
| ok = ~torch.isnan(y) | |
| if ok.any(): | |
| terms["score"] = F.binary_cross_entropy_with_logits(score_logit[ok], y[ok]) | |
| total = total + w_score * terms["score"] | |
| return total, {k: float(v.detach()) for k, v in terms.items()} | |