Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/services/training_service.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 5.07 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/training_service.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/services/training_service.py
-
curl -L -o training_service.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/training_service.py
5.07 kB
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| from torch.utils.data import DataLoader | |
| from src.models.config import GrassmannConfig | |
| from src.models.grassmann_model import GrassmannTranscriptionModel | |
| class TrainMetrics: | |
| epoch: int | |
| train_loss: float | |
| val_loss: float | None = None | |
| class TrainingService: | |
| """Handles the training loop, loss computation, and optimization.""" | |
| def __init__( | |
| self, | |
| model: GrassmannTranscriptionModel, | |
| config: GrassmannConfig, | |
| device: torch.device | None = None, | |
| ): | |
| self.model = model | |
| self.config = config | |
| if device is not None: | |
| self.device = device | |
| elif torch.cuda.is_available(): | |
| self.device = torch.device("cuda") | |
| else: | |
| # MPS can be unstable with some PyTorch/Python combos; prefer CPU as fallback | |
| self.device = torch.device("cpu") | |
| self.model.to(self.device) | |
| self.optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=config.learning_rate, | |
| weight_decay=config.weight_decay, | |
| ) | |
| self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None | |
| self.ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True) | |
| self.history: list[TrainMetrics] = [] | |
| def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None: | |
| """Add cosine annealing LR schedule with linear warmup.""" | |
| from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR | |
| warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs) | |
| cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6) | |
| self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs]) | |
| def train_epoch(self, train_loader: DataLoader) -> float: | |
| self.model.train() | |
| total_loss = 0.0 | |
| n_batches = 0 | |
| total_batches = len(train_loader) | |
| for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader): | |
| mels = mels.to(self.device) | |
| tokens = tokens.to(self.device) | |
| mel_lengths = mel_lengths.to(self.device) | |
| token_lengths = token_lengths.to(self.device) | |
| self.optimizer.zero_grad() | |
| log_probs = self.model(mels) # (T', B, vocab) | |
| output_lengths = self.model.get_output_lengths(mel_lengths) | |
| loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths) | |
| if torch.isfinite(loss): | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0) | |
| self.optimizer.step() | |
| total_loss += loss.item() | |
| n_batches += 1 | |
| if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches: | |
| avg = total_loss / max(n_batches, 1) | |
| print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True) | |
| return total_loss / max(n_batches, 1) | |
| def validate(self, val_loader: DataLoader) -> float: | |
| self.model.eval() | |
| total_loss = 0.0 | |
| n_batches = 0 | |
| for mels, tokens, mel_lengths, token_lengths in val_loader: | |
| mels = mels.to(self.device) | |
| tokens = tokens.to(self.device) | |
| mel_lengths = mel_lengths.to(self.device) | |
| token_lengths = token_lengths.to(self.device) | |
| log_probs = self.model(mels) | |
| output_lengths = self.model.get_output_lengths(mel_lengths) | |
| loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths) | |
| if torch.isfinite(loss): | |
| total_loss += loss.item() | |
| n_batches += 1 | |
| return total_loss / max(n_batches, 1) | |
| def train_loop( | |
| self, | |
| train_loader: DataLoader, | |
| val_loader: DataLoader | None, | |
| epochs: int, | |
| callback: object | None = None, | |
| ) -> list[TrainMetrics]: | |
| for epoch in range(1, epochs + 1): | |
| train_loss = self.train_epoch(train_loader) | |
| val_loss = self.validate(val_loader) if val_loader else None | |
| metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss) | |
| self.history.append(metrics) | |
| if callback and hasattr(callback, "on_epoch_end"): | |
| callback.on_epoch_end(metrics) | |
| if self.scheduler is not None: | |
| self.scheduler.step() | |
| return self.history | |
| def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor: | |
| """Greedy CTC decoding for a single mel spectrogram.""" | |
| self.model.eval() | |
| if mel.dim() == 2: | |
| mel = mel.unsqueeze(0) # add batch dim | |
| mel = mel.to(self.device) | |
| log_probs = self.model(mel) # (T', 1, vocab) | |
| pred = log_probs.argmax(dim=-1).squeeze(1) # (T',) | |
| return pred | |