Download scripts/duplex/evaluation/loss.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 5.33 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/evaluation/loss.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/evaluation/loss.py
-
curl -L -o loss.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/evaluation/loss.py
5.33 kB
| """Float32 cross entropy, the weighted training loss and additive unweighted statistics.""" | |
| import math | |
| import torch | |
| from torch.nn import functional as F | |
| # Moshi's equation 7 (Défossez et al. 2024): each track's codebook 0 against 1 for | |
| # each acoustic codebook, and its padding weight for the advance token. Kept equal | |
| # to duplex.model.network, which this module must not import. | |
| SEMANTIC_WEIGHT = 100.0 | |
| ADVANCE_WEIGHT = 0.5 | |
| ADVANCE = 32000 | |
| def frame_nll(logits, targets): | |
| return F.cross_entropy( | |
| logits.float().reshape(-1, logits.shape[-1]), | |
| targets.reshape(-1), | |
| reduction="none", | |
| ).reshape(targets.shape) | |
| def loss_and_stats( | |
| logits, | |
| targets, | |
| scored, | |
| fps=12.5, | |
| track_codebooks=8, | |
| text_logits=None, | |
| text_targets=None, | |
| text_weights=None, | |
| semantic_weight=SEMANTIC_WEIGHT, | |
| text_tracks=None, | |
| nll=frame_nll, | |
| reduce=None, | |
| ): | |
| """Training loss and unweighted statistics. | |
| scored is the boolean [batch, frames, codebooks] mask of real tokens on present | |
| tracks; codebooks run track by track in groups of track_codebooks. The loss is | |
| the weighted mean audio token loss (codebook 0 of each track at semantic_weight) | |
| plus, when text is given, the weighted mean text event loss, as in Moshi. nll is | |
| frame_nll or a compiled copy, which never stores the float32 logits. Under data | |
| parallelism, reduce sums a tensor over the processes: each process's loss then | |
| divides by the whole batch's weights, so the processes' losses and gradients sum | |
| to those of the whole batch. | |
| """ | |
| if targets.shape[-1] % track_codebooks: | |
| raise ValueError("Codebooks must fill whole tracks") | |
| if scored.shape != targets.shape or scored.dtype != torch.bool: | |
| raise ValueError("Use one boolean scoring mask per token") | |
| frames = scored.any(-1) | |
| count = int(frames.sum()) | |
| if not count: | |
| raise ValueError("No scored frames") | |
| token_nll = nll(logits, targets) | |
| q = targets.shape[-1] | |
| weights = torch.ones(q, device=token_nll.device) | |
| weights[::track_codebooks] = semantic_weight | |
| w = scored * weights | |
| totals = torch.stack( | |
| (w.sum(), text_weights.sum() if text_logits is not None else w.new_zeros(())) | |
| ).detach() | |
| if reduce is not None: | |
| totals = reduce(totals) | |
| loss = (token_nll * w).sum() / totals[0] | |
| masked = (token_nll * scored).detach().double() | |
| codebooks = masked.sum((0, 1)) | |
| tracks = codebooks.reshape(-1, track_codebooks) | |
| stats = { | |
| "nats": codebooks.sum().item(), | |
| # Codebook 0 of every track. | |
| "first_codebook_nats": tracks[:, 0].sum().item(), | |
| "track_nats": tracks.sum(1).tolist(), | |
| "first_codebook_tokens": int(scored[..., ::track_codebooks].sum()), | |
| "frames": count, | |
| "seconds": count / fps, | |
| "text_nats": 0.0, | |
| "text_events": 0, | |
| } | |
| if text_logits is not None: | |
| text_nll = nll(text_logits, text_targets) | |
| if totals[1] > 0: | |
| loss = loss + (text_nll * text_weights).sum() / totals[1] | |
| counted = text_weights > 0 | |
| stats["text_nats"] = text_nll[counted].double().sum().item() | |
| stats["text_events"] = int(counted.sum()) | |
| if text_tracks is not None: | |
| words = counted & (text_targets != ADVANCE) | |
| on = [text_tracks == j for j in range(targets.shape[-1] // track_codebooks)] | |
| stats["track_text_nats"] = [ | |
| text_nll[counted & t].double().sum().item() for t in on | |
| ] | |
| stats["track_word_nats"] = [ | |
| text_nll[words & t].double().sum().item() for t in on | |
| ] | |
| stats["track_words"] = [int((words & t).sum()) for t in on] | |
| return loss, stats | |
| def summarize(stats): | |
| nats = sum(s["nats"] for s in stats) | |
| first = sum(s["first_codebook_nats"] for s in stats) | |
| seconds = sum(s["seconds"] for s in stats) | |
| if seconds <= 0: | |
| raise ValueError("No scored audio") | |
| bits = math.log(2) * seconds | |
| result = { | |
| "bits_per_second": nats / bits, | |
| "first_codebook_bits_per_second": first / bits, | |
| "track_bits_per_second": [ | |
| sum(s["track_nats"][j] for s in stats) / bits | |
| for j in range(len(stats[0]["track_nats"])) | |
| ], | |
| "scored_seconds": seconds, | |
| "scored_frames": sum(s["frames"] for s in stats), | |
| } | |
| events = sum(s.get("text_events", 0) for s in stats) | |
| if events: | |
| result["text_bits_per_second"] = sum(s["text_nats"] for s in stats) / bits | |
| result["text_events"] = events | |
| tracked = [s for s in stats if "track_text_nats" in s] | |
| if tracked: | |
| tracks = len(tracked[0]["track_text_nats"]) | |
| def total(key, j): | |
| return sum(s[key][j] for s in tracked) | |
| result["track_text_bits_per_second"] = [ | |
| total("track_text_nats", j) / bits for j in range(tracks) | |
| ] | |
| # Word tokens alone, without advance events: bits per word token. | |
| result["track_word_bits_per_token"] = [ | |
| total("track_word_nats", j) / math.log(2) / total("track_words", j) | |
| if total("track_words", j) | |
| else None | |
| for j in range(tracks) | |
| ] | |
| result["track_words"] = [total("track_words", j) for j in range(tracks)] | |
| return result | |