"""Float32 cross entropy, the weighted training loss and additive unweighted statistics.""" import math import torch from torch.nn import functional as F # Moshi's equation 7 (Défossez et al. 2024): each track's codebook 0 against 1 for # each acoustic codebook, and its padding weight for the advance token. Kept equal # to duplex.model.network, which this module must not import. SEMANTIC_WEIGHT = 100.0 ADVANCE_WEIGHT = 0.5 ADVANCE = 32000 def frame_nll(logits, targets): return F.cross_entropy( logits.float().reshape(-1, logits.shape[-1]), targets.reshape(-1), reduction="none", ).reshape(targets.shape) def loss_and_stats( logits, targets, scored, fps=12.5, track_codebooks=8, text_logits=None, text_targets=None, text_weights=None, semantic_weight=SEMANTIC_WEIGHT, text_tracks=None, nll=frame_nll, reduce=None, ): """Training loss and unweighted statistics. scored is the boolean [batch, frames, codebooks] mask of real tokens on present tracks; codebooks run track by track in groups of track_codebooks. The loss is the weighted mean audio token loss (codebook 0 of each track at semantic_weight) plus, when text is given, the weighted mean text event loss, as in Moshi. nll is frame_nll or a compiled copy, which never stores the float32 logits. Under data parallelism, reduce sums a tensor over the processes: each process's loss then divides by the whole batch's weights, so the processes' losses and gradients sum to those of the whole batch. """ if targets.shape[-1] % track_codebooks: raise ValueError("Codebooks must fill whole tracks") if scored.shape != targets.shape or scored.dtype != torch.bool: raise ValueError("Use one boolean scoring mask per token") frames = scored.any(-1) count = int(frames.sum()) if not count: raise ValueError("No scored frames") token_nll = nll(logits, targets) q = targets.shape[-1] weights = torch.ones(q, device=token_nll.device) weights[::track_codebooks] = semantic_weight w = scored * weights totals = torch.stack( (w.sum(), text_weights.sum() if text_logits is not None else w.new_zeros(())) ).detach() if reduce is not None: totals = reduce(totals) loss = (token_nll * w).sum() / totals[0] masked = (token_nll * scored).detach().double() codebooks = masked.sum((0, 1)) tracks = codebooks.reshape(-1, track_codebooks) stats = { "nats": codebooks.sum().item(), # Codebook 0 of every track. "first_codebook_nats": tracks[:, 0].sum().item(), "track_nats": tracks.sum(1).tolist(), "first_codebook_tokens": int(scored[..., ::track_codebooks].sum()), "frames": count, "seconds": count / fps, "text_nats": 0.0, "text_events": 0, } if text_logits is not None: text_nll = nll(text_logits, text_targets) if totals[1] > 0: loss = loss + (text_nll * text_weights).sum() / totals[1] counted = text_weights > 0 stats["text_nats"] = text_nll[counted].double().sum().item() stats["text_events"] = int(counted.sum()) if text_tracks is not None: words = counted & (text_targets != ADVANCE) on = [text_tracks == j for j in range(targets.shape[-1] // track_codebooks)] stats["track_text_nats"] = [ text_nll[counted & t].double().sum().item() for t in on ] stats["track_word_nats"] = [ text_nll[words & t].double().sum().item() for t in on ] stats["track_words"] = [int((words & t).sum()) for t in on] return loss, stats def summarize(stats): nats = sum(s["nats"] for s in stats) first = sum(s["first_codebook_nats"] for s in stats) seconds = sum(s["seconds"] for s in stats) if seconds <= 0: raise ValueError("No scored audio") bits = math.log(2) * seconds result = { "bits_per_second": nats / bits, "first_codebook_bits_per_second": first / bits, "track_bits_per_second": [ sum(s["track_nats"][j] for s in stats) / bits for j in range(len(stats[0]["track_nats"])) ], "scored_seconds": seconds, "scored_frames": sum(s["frames"] for s in stats), } events = sum(s.get("text_events", 0) for s in stats) if events: result["text_bits_per_second"] = sum(s["text_nats"] for s in stats) / bits result["text_events"] = events tracked = [s for s in stats if "track_text_nats" in s] if tracked: tracks = len(tracked[0]["track_text_nats"]) def total(key, j): return sum(s[key][j] for s in tracked) result["track_text_bits_per_second"] = [ total("track_text_nats", j) / bits for j in range(tracks) ] # Word tokens alone, without advance events: bits per word token. result["track_word_bits_per_token"] = [ total("track_word_nats", j) / math.log(2) / total("track_words", j) if total("track_words", j) else None for j in range(tracks) ] result["track_words"] = [total("track_words", j) for j in range(tracks)] return result