grassmann-cache / src /services /seq2seq_training_service.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
4.98 kB
"""
Seq2Seq Training Service.
Handles teacher-forced training for the Grassmann encoder-decoder model.
Uses cross-entropy loss instead of CTC, with proper token shifting
and padding masks.
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from src.models.config import GrassmannConfig
from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
@dataclass
class TrainMetrics:
epoch: int
train_loss: float
val_loss: float | None = None
class Seq2SeqTrainingService:
"""Training loop for the fully attention-free encoder-decoder."""
def __init__(
self,
model: GrassmannSeq2SeqModel,
config: GrassmannConfig,
device: torch.device | None = None,
):
self.model = model
self.config = config
if device is not None:
self.device = device
elif torch.cuda.is_available():
self.device = torch.device("cuda")
else:
self.device = torch.device("cpu")
self.model.to(self.device)
self.optimizer = torch.optim.AdamW(
model.parameters(),
lr=config.learning_rate,
weight_decay=config.weight_decay,
)
self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
# Cross-entropy loss, ignoring padding (index 0)
self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
self.history: list[TrainMetrics] = []
def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
def train_epoch(self, train_loader: DataLoader) -> float:
self.model.train()
total_loss = 0.0
n_batches = 0
total_batches = len(train_loader)
for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
mels = mels.to(self.device)
tokens = tokens.to(self.device)
self.optimizer.zero_grad()
# Teacher-forced forward pass
logits = self.model(mels, tokens) # (B, T_dec, vocab)
# Reshape for cross-entropy: (B*T, vocab) vs (B*T,)
B, T, V = logits.shape
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
if torch.isfinite(loss):
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
self.optimizer.step()
total_loss += loss.item()
n_batches += 1
# Log every 10 batches for CI visibility
if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
avg = total_loss / max(n_batches, 1)
print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
return total_loss / max(n_batches, 1)
@torch.no_grad()
def validate(self, val_loader: DataLoader) -> float:
self.model.eval()
total_loss = 0.0
n_batches = 0
for mels, tokens, mel_lengths, token_lengths in val_loader:
mels = mels.to(self.device)
tokens = tokens.to(self.device)
logits = self.model(mels, tokens)
B, T, V = logits.shape
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
if torch.isfinite(loss):
total_loss += loss.item()
n_batches += 1
return total_loss / max(n_batches, 1)
def train_loop(
self,
train_loader: DataLoader,
val_loader: DataLoader | None,
epochs: int,
callback: object | None = None,
) -> list[TrainMetrics]:
for epoch in range(1, epochs + 1):
train_loss = self.train_epoch(train_loader)
val_loss = self.validate(val_loader) if val_loader else None
metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
self.history.append(metrics)
if callback and hasattr(callback, "on_epoch_end"):
callback.on_epoch_end(metrics)
if self.scheduler is not None:
self.scheduler.step()
return self.history
@torch.no_grad()
def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
"""Autoregressive greedy decoding for a single mel spectrogram."""
self.model.eval()
if mel.dim() == 2:
mel = mel.unsqueeze(0)
mel = mel.to(self.device)
return self.model.greedy_decode(mel, max_len=200)