Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/services/seq2seq_training_service.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 4.98 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/seq2seq_training_service.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/services/seq2seq_training_service.py
-
curl -L -o seq2seq_training_service.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/services/seq2seq_training_service.py
4.98 kB
| """ | |
| Seq2Seq Training Service. | |
| Handles teacher-forced training for the Grassmann encoder-decoder model. | |
| Uses cross-entropy loss instead of CTC, with proper token shifting | |
| and padding masks. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| from torch.utils.data import DataLoader | |
| from src.models.config import GrassmannConfig | |
| from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel | |
| class TrainMetrics: | |
| epoch: int | |
| train_loss: float | |
| val_loss: float | None = None | |
| class Seq2SeqTrainingService: | |
| """Training loop for the fully attention-free encoder-decoder.""" | |
| def __init__( | |
| self, | |
| model: GrassmannSeq2SeqModel, | |
| config: GrassmannConfig, | |
| device: torch.device | None = None, | |
| ): | |
| self.model = model | |
| self.config = config | |
| if device is not None: | |
| self.device = device | |
| elif torch.cuda.is_available(): | |
| self.device = torch.device("cuda") | |
| else: | |
| self.device = torch.device("cpu") | |
| self.model.to(self.device) | |
| self.optimizer = torch.optim.AdamW( | |
| model.parameters(), | |
| lr=config.learning_rate, | |
| weight_decay=config.weight_decay, | |
| ) | |
| self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None | |
| # Cross-entropy loss, ignoring padding (index 0) | |
| self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1) | |
| self.history: list[TrainMetrics] = [] | |
| def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None: | |
| from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR | |
| warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs) | |
| cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6) | |
| self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs]) | |
| def train_epoch(self, train_loader: DataLoader) -> float: | |
| self.model.train() | |
| total_loss = 0.0 | |
| n_batches = 0 | |
| total_batches = len(train_loader) | |
| for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader): | |
| mels = mels.to(self.device) | |
| tokens = tokens.to(self.device) | |
| self.optimizer.zero_grad() | |
| # Teacher-forced forward pass | |
| logits = self.model(mels, tokens) # (B, T_dec, vocab) | |
| # Reshape for cross-entropy: (B*T, vocab) vs (B*T,) | |
| B, T, V = logits.shape | |
| loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T)) | |
| if torch.isfinite(loss): | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0) | |
| self.optimizer.step() | |
| total_loss += loss.item() | |
| n_batches += 1 | |
| # Log every 10 batches for CI visibility | |
| if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches: | |
| avg = total_loss / max(n_batches, 1) | |
| print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True) | |
| return total_loss / max(n_batches, 1) | |
| def validate(self, val_loader: DataLoader) -> float: | |
| self.model.eval() | |
| total_loss = 0.0 | |
| n_batches = 0 | |
| for mels, tokens, mel_lengths, token_lengths in val_loader: | |
| mels = mels.to(self.device) | |
| tokens = tokens.to(self.device) | |
| logits = self.model(mels, tokens) | |
| B, T, V = logits.shape | |
| loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T)) | |
| if torch.isfinite(loss): | |
| total_loss += loss.item() | |
| n_batches += 1 | |
| return total_loss / max(n_batches, 1) | |
| def train_loop( | |
| self, | |
| train_loader: DataLoader, | |
| val_loader: DataLoader | None, | |
| epochs: int, | |
| callback: object | None = None, | |
| ) -> list[TrainMetrics]: | |
| for epoch in range(1, epochs + 1): | |
| train_loss = self.train_epoch(train_loader) | |
| val_loss = self.validate(val_loader) if val_loader else None | |
| metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss) | |
| self.history.append(metrics) | |
| if callback and hasattr(callback, "on_epoch_end"): | |
| callback.on_epoch_end(metrics) | |
| if self.scheduler is not None: | |
| self.scheduler.step() | |
| return self.history | |
| def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor: | |
| """Autoregressive greedy decoding for a single mel spectrogram.""" | |
| self.model.eval() | |
| if mel.dim() == 2: | |
| mel = mel.unsqueeze(0) | |
| mel = mel.to(self.device) | |
| return self.model.greedy_decode(mel, max_len=200) | |