Spaces:
Running
Running
File size: 4,976 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """
Seq2Seq Training Service.
Handles teacher-forced training for the Grassmann encoder-decoder model.
Uses cross-entropy loss instead of CTC, with proper token shifting
and padding masks.
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from src.models.config import GrassmannConfig
from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
@dataclass
class TrainMetrics:
epoch: int
train_loss: float
val_loss: float | None = None
class Seq2SeqTrainingService:
"""Training loop for the fully attention-free encoder-decoder."""
def __init__(
self,
model: GrassmannSeq2SeqModel,
config: GrassmannConfig,
device: torch.device | None = None,
):
self.model = model
self.config = config
if device is not None:
self.device = device
elif torch.cuda.is_available():
self.device = torch.device("cuda")
else:
self.device = torch.device("cpu")
self.model.to(self.device)
self.optimizer = torch.optim.AdamW(
model.parameters(),
lr=config.learning_rate,
weight_decay=config.weight_decay,
)
self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
# Cross-entropy loss, ignoring padding (index 0)
self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
self.history: list[TrainMetrics] = []
def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
def train_epoch(self, train_loader: DataLoader) -> float:
self.model.train()
total_loss = 0.0
n_batches = 0
total_batches = len(train_loader)
for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
mels = mels.to(self.device)
tokens = tokens.to(self.device)
self.optimizer.zero_grad()
# Teacher-forced forward pass
logits = self.model(mels, tokens) # (B, T_dec, vocab)
# Reshape for cross-entropy: (B*T, vocab) vs (B*T,)
B, T, V = logits.shape
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
if torch.isfinite(loss):
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
self.optimizer.step()
total_loss += loss.item()
n_batches += 1
# Log every 10 batches for CI visibility
if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
avg = total_loss / max(n_batches, 1)
print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
return total_loss / max(n_batches, 1)
@torch.no_grad()
def validate(self, val_loader: DataLoader) -> float:
self.model.eval()
total_loss = 0.0
n_batches = 0
for mels, tokens, mel_lengths, token_lengths in val_loader:
mels = mels.to(self.device)
tokens = tokens.to(self.device)
logits = self.model(mels, tokens)
B, T, V = logits.shape
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
if torch.isfinite(loss):
total_loss += loss.item()
n_batches += 1
return total_loss / max(n_batches, 1)
def train_loop(
self,
train_loader: DataLoader,
val_loader: DataLoader | None,
epochs: int,
callback: object | None = None,
) -> list[TrainMetrics]:
for epoch in range(1, epochs + 1):
train_loss = self.train_epoch(train_loader)
val_loss = self.validate(val_loader) if val_loader else None
metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
self.history.append(metrics)
if callback and hasattr(callback, "on_epoch_end"):
callback.on_epoch_end(metrics)
if self.scheduler is not None:
self.scheduler.step()
return self.history
@torch.no_grad()
def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
"""Autoregressive greedy decoding for a single mel spectrogram."""
self.model.eval()
if mel.dim() == 2:
mel = mel.unsqueeze(0)
mel = mel.to(self.device)
return self.model.greedy_decode(mel, max_len=200)
|