grassmann-cache / src /services /training_service.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
5.07 kB
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from src.models.config import GrassmannConfig
from src.models.grassmann_model import GrassmannTranscriptionModel
@dataclass
class TrainMetrics:
epoch: int
train_loss: float
val_loss: float | None = None
class TrainingService:
"""Handles the training loop, loss computation, and optimization."""
def __init__(
self,
model: GrassmannTranscriptionModel,
config: GrassmannConfig,
device: torch.device | None = None,
):
self.model = model
self.config = config
if device is not None:
self.device = device
elif torch.cuda.is_available():
self.device = torch.device("cuda")
else:
# MPS can be unstable with some PyTorch/Python combos; prefer CPU as fallback
self.device = torch.device("cpu")
self.model.to(self.device)
self.optimizer = torch.optim.AdamW(
model.parameters(),
lr=config.learning_rate,
weight_decay=config.weight_decay,
)
self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
self.ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True)
self.history: list[TrainMetrics] = []
def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
"""Add cosine annealing LR schedule with linear warmup."""
from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
def train_epoch(self, train_loader: DataLoader) -> float:
self.model.train()
total_loss = 0.0
n_batches = 0
total_batches = len(train_loader)
for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
mels = mels.to(self.device)
tokens = tokens.to(self.device)
mel_lengths = mel_lengths.to(self.device)
token_lengths = token_lengths.to(self.device)
self.optimizer.zero_grad()
log_probs = self.model(mels) # (T', B, vocab)
output_lengths = self.model.get_output_lengths(mel_lengths)
loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
if torch.isfinite(loss):
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
self.optimizer.step()
total_loss += loss.item()
n_batches += 1
if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
avg = total_loss / max(n_batches, 1)
print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
return total_loss / max(n_batches, 1)
@torch.no_grad()
def validate(self, val_loader: DataLoader) -> float:
self.model.eval()
total_loss = 0.0
n_batches = 0
for mels, tokens, mel_lengths, token_lengths in val_loader:
mels = mels.to(self.device)
tokens = tokens.to(self.device)
mel_lengths = mel_lengths.to(self.device)
token_lengths = token_lengths.to(self.device)
log_probs = self.model(mels)
output_lengths = self.model.get_output_lengths(mel_lengths)
loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
if torch.isfinite(loss):
total_loss += loss.item()
n_batches += 1
return total_loss / max(n_batches, 1)
def train_loop(
self,
train_loader: DataLoader,
val_loader: DataLoader | None,
epochs: int,
callback: object | None = None,
) -> list[TrainMetrics]:
for epoch in range(1, epochs + 1):
train_loss = self.train_epoch(train_loader)
val_loss = self.validate(val_loader) if val_loader else None
metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
self.history.append(metrics)
if callback and hasattr(callback, "on_epoch_end"):
callback.on_epoch_end(metrics)
if self.scheduler is not None:
self.scheduler.step()
return self.history
@torch.no_grad()
def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
"""Greedy CTC decoding for a single mel spectrogram."""
self.model.eval()
if mel.dim() == 2:
mel = mel.unsqueeze(0) # add batch dim
mel = mel.to(self.device)
log_probs = self.model(mel) # (T', 1, vocab)
pred = log_probs.argmax(dim=-1).squeeze(1) # (T',)
return pred