Safetensors
English
RonanMcGovern's picture
Scripts under the duplex package name; card final pass
d90680b verified
Raw History Blame Contribute Delete
5.33 kB
"""Float32 cross entropy, the weighted training loss and additive unweighted statistics."""
import math
import torch
from torch.nn import functional as F
# Moshi's equation 7 (Défossez et al. 2024): each track's codebook 0 against 1 for
# each acoustic codebook, and its padding weight for the advance token. Kept equal
# to duplex.model.network, which this module must not import.
SEMANTIC_WEIGHT = 100.0
ADVANCE_WEIGHT = 0.5
ADVANCE = 32000
def frame_nll(logits, targets):
return F.cross_entropy(
logits.float().reshape(-1, logits.shape[-1]),
targets.reshape(-1),
reduction="none",
).reshape(targets.shape)
def loss_and_stats(
logits,
targets,
scored,
fps=12.5,
track_codebooks=8,
text_logits=None,
text_targets=None,
text_weights=None,
semantic_weight=SEMANTIC_WEIGHT,
text_tracks=None,
nll=frame_nll,
reduce=None,
):
"""Training loss and unweighted statistics.
scored is the boolean [batch, frames, codebooks] mask of real tokens on present
tracks; codebooks run track by track in groups of track_codebooks. The loss is
the weighted mean audio token loss (codebook 0 of each track at semantic_weight)
plus, when text is given, the weighted mean text event loss, as in Moshi. nll is
frame_nll or a compiled copy, which never stores the float32 logits. Under data
parallelism, reduce sums a tensor over the processes: each process's loss then
divides by the whole batch's weights, so the processes' losses and gradients sum
to those of the whole batch.
"""
if targets.shape[-1] % track_codebooks:
raise ValueError("Codebooks must fill whole tracks")
if scored.shape != targets.shape or scored.dtype != torch.bool:
raise ValueError("Use one boolean scoring mask per token")
frames = scored.any(-1)
count = int(frames.sum())
if not count:
raise ValueError("No scored frames")
token_nll = nll(logits, targets)
q = targets.shape[-1]
weights = torch.ones(q, device=token_nll.device)
weights[::track_codebooks] = semantic_weight
w = scored * weights
totals = torch.stack(
(w.sum(), text_weights.sum() if text_logits is not None else w.new_zeros(()))
).detach()
if reduce is not None:
totals = reduce(totals)
loss = (token_nll * w).sum() / totals[0]
masked = (token_nll * scored).detach().double()
codebooks = masked.sum((0, 1))
tracks = codebooks.reshape(-1, track_codebooks)
stats = {
"nats": codebooks.sum().item(),
# Codebook 0 of every track.
"first_codebook_nats": tracks[:, 0].sum().item(),
"track_nats": tracks.sum(1).tolist(),
"first_codebook_tokens": int(scored[..., ::track_codebooks].sum()),
"frames": count,
"seconds": count / fps,
"text_nats": 0.0,
"text_events": 0,
}
if text_logits is not None:
text_nll = nll(text_logits, text_targets)
if totals[1] > 0:
loss = loss + (text_nll * text_weights).sum() / totals[1]
counted = text_weights > 0
stats["text_nats"] = text_nll[counted].double().sum().item()
stats["text_events"] = int(counted.sum())
if text_tracks is not None:
words = counted & (text_targets != ADVANCE)
on = [text_tracks == j for j in range(targets.shape[-1] // track_codebooks)]
stats["track_text_nats"] = [
text_nll[counted & t].double().sum().item() for t in on
]
stats["track_word_nats"] = [
text_nll[words & t].double().sum().item() for t in on
]
stats["track_words"] = [int((words & t).sum()) for t in on]
return loss, stats
def summarize(stats):
nats = sum(s["nats"] for s in stats)
first = sum(s["first_codebook_nats"] for s in stats)
seconds = sum(s["seconds"] for s in stats)
if seconds <= 0:
raise ValueError("No scored audio")
bits = math.log(2) * seconds
result = {
"bits_per_second": nats / bits,
"first_codebook_bits_per_second": first / bits,
"track_bits_per_second": [
sum(s["track_nats"][j] for s in stats) / bits
for j in range(len(stats[0]["track_nats"]))
],
"scored_seconds": seconds,
"scored_frames": sum(s["frames"] for s in stats),
}
events = sum(s.get("text_events", 0) for s in stats)
if events:
result["text_bits_per_second"] = sum(s["text_nats"] for s in stats) / bits
result["text_events"] = events
tracked = [s for s in stats if "track_text_nats" in s]
if tracked:
tracks = len(tracked[0]["track_text_nats"])
def total(key, j):
return sum(s[key][j] for s in tracked)
result["track_text_bits_per_second"] = [
total("track_text_nats", j) / bits for j in range(tracks)
]
# Word tokens alone, without advance events: bits per word token.
result["track_word_bits_per_token"] = [
total("track_word_nats", j) / math.log(2) / total("track_words", j)
if total("track_words", j)
else None
for j in range(tracks)
]
result["track_words"] = [total("track_words", j) for j in range(tracks)]
return result