File size: 5,329 Bytes
d52c89c d90680b d52c89c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | """Float32 cross entropy, the weighted training loss and additive unweighted statistics."""
import math
import torch
from torch.nn import functional as F
# Moshi's equation 7 (Défossez et al. 2024): each track's codebook 0 against 1 for
# each acoustic codebook, and its padding weight for the advance token. Kept equal
# to duplex.model.network, which this module must not import.
SEMANTIC_WEIGHT = 100.0
ADVANCE_WEIGHT = 0.5
ADVANCE = 32000
def frame_nll(logits, targets):
return F.cross_entropy(
logits.float().reshape(-1, logits.shape[-1]),
targets.reshape(-1),
reduction="none",
).reshape(targets.shape)
def loss_and_stats(
logits,
targets,
scored,
fps=12.5,
track_codebooks=8,
text_logits=None,
text_targets=None,
text_weights=None,
semantic_weight=SEMANTIC_WEIGHT,
text_tracks=None,
nll=frame_nll,
reduce=None,
):
"""Training loss and unweighted statistics.
scored is the boolean [batch, frames, codebooks] mask of real tokens on present
tracks; codebooks run track by track in groups of track_codebooks. The loss is
the weighted mean audio token loss (codebook 0 of each track at semantic_weight)
plus, when text is given, the weighted mean text event loss, as in Moshi. nll is
frame_nll or a compiled copy, which never stores the float32 logits. Under data
parallelism, reduce sums a tensor over the processes: each process's loss then
divides by the whole batch's weights, so the processes' losses and gradients sum
to those of the whole batch.
"""
if targets.shape[-1] % track_codebooks:
raise ValueError("Codebooks must fill whole tracks")
if scored.shape != targets.shape or scored.dtype != torch.bool:
raise ValueError("Use one boolean scoring mask per token")
frames = scored.any(-1)
count = int(frames.sum())
if not count:
raise ValueError("No scored frames")
token_nll = nll(logits, targets)
q = targets.shape[-1]
weights = torch.ones(q, device=token_nll.device)
weights[::track_codebooks] = semantic_weight
w = scored * weights
totals = torch.stack(
(w.sum(), text_weights.sum() if text_logits is not None else w.new_zeros(()))
).detach()
if reduce is not None:
totals = reduce(totals)
loss = (token_nll * w).sum() / totals[0]
masked = (token_nll * scored).detach().double()
codebooks = masked.sum((0, 1))
tracks = codebooks.reshape(-1, track_codebooks)
stats = {
"nats": codebooks.sum().item(),
# Codebook 0 of every track.
"first_codebook_nats": tracks[:, 0].sum().item(),
"track_nats": tracks.sum(1).tolist(),
"first_codebook_tokens": int(scored[..., ::track_codebooks].sum()),
"frames": count,
"seconds": count / fps,
"text_nats": 0.0,
"text_events": 0,
}
if text_logits is not None:
text_nll = nll(text_logits, text_targets)
if totals[1] > 0:
loss = loss + (text_nll * text_weights).sum() / totals[1]
counted = text_weights > 0
stats["text_nats"] = text_nll[counted].double().sum().item()
stats["text_events"] = int(counted.sum())
if text_tracks is not None:
words = counted & (text_targets != ADVANCE)
on = [text_tracks == j for j in range(targets.shape[-1] // track_codebooks)]
stats["track_text_nats"] = [
text_nll[counted & t].double().sum().item() for t in on
]
stats["track_word_nats"] = [
text_nll[words & t].double().sum().item() for t in on
]
stats["track_words"] = [int((words & t).sum()) for t in on]
return loss, stats
def summarize(stats):
nats = sum(s["nats"] for s in stats)
first = sum(s["first_codebook_nats"] for s in stats)
seconds = sum(s["seconds"] for s in stats)
if seconds <= 0:
raise ValueError("No scored audio")
bits = math.log(2) * seconds
result = {
"bits_per_second": nats / bits,
"first_codebook_bits_per_second": first / bits,
"track_bits_per_second": [
sum(s["track_nats"][j] for s in stats) / bits
for j in range(len(stats[0]["track_nats"]))
],
"scored_seconds": seconds,
"scored_frames": sum(s["frames"] for s in stats),
}
events = sum(s.get("text_events", 0) for s in stats)
if events:
result["text_bits_per_second"] = sum(s["text_nats"] for s in stats) / bits
result["text_events"] = events
tracked = [s for s in stats if "track_text_nats" in s]
if tracked:
tracks = len(tracked[0]["track_text_nats"])
def total(key, j):
return sum(s[key][j] for s in tracked)
result["track_text_bits_per_second"] = [
total("track_text_nats", j) / bits for j in range(tracks)
]
# Word tokens alone, without advance events: bits per word token.
result["track_word_bits_per_token"] = [
total("track_word_nats", j) / math.log(2) / total("track_words", j)
if total("track_words", j)
else None
for j in range(tracks)
]
result["track_words"] = [total("track_words", j) for j in range(tracks)]
return result
|