Spaces:
Running
Running
Nicholas Finch commited on
Commit ·
d7013e6
0
Parent(s):
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
Browse files- Dockerfile +26 -0
- README.md +10 -0
- app.py +518 -0
- recipes.yaml +60 -0
- requirements.txt +7 -0
- src/__init__.py +0 -0
- src/controllers/__init__.py +2 -0
- src/controllers/model_controller.py +93 -0
- src/controllers/pipeline_controller.py +102 -0
- src/interfaces/__init__.py +4 -0
- src/interfaces/i_audio.py +23 -0
- src/interfaces/i_dataset.py +40 -0
- src/interfaces/i_model.py +43 -0
- src/interfaces/i_tokenizer.py +28 -0
- src/models/__init__.py +9 -0
- src/models/ablation_layer.py +175 -0
- src/models/config.py +58 -0
- src/models/conformer_block.py +148 -0
- src/models/conv_module.py +52 -0
- src/models/grassmann_cross_layer.py +126 -0
- src/models/grassmann_decoder.py +130 -0
- src/models/grassmann_layer.py +186 -0
- src/models/grassmann_model.py +179 -0
- src/models/grassmann_seq2seq.py +183 -0
- src/repositories/__init__.py +2 -0
- src/repositories/cache_repository.py +267 -0
- src/repositories/dataset_repository.py +286 -0
- src/services/__init__.py +3 -0
- src/services/audio_service.py +43 -0
- src/services/dataset_service.py +295 -0
- src/services/eval_service.py +124 -0
- src/services/export_service.py +129 -0
- src/services/hf_dataset_service.py +657 -0
- src/services/model_hub_service.py +86 -0
- src/services/seq2seq_training_service.py +149 -0
- src/services/tokenizer_service.py +172 -0
- src/services/training_service.py +141 -0
- src/services/upload_service.py +173 -0
- src/views/__init__.py +1 -0
- src/views/console_view.py +50 -0
Dockerfile
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.11-slim
|
| 2 |
+
|
| 3 |
+
WORKDIR /app
|
| 4 |
+
|
| 5 |
+
# System deps for audio decoding
|
| 6 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 7 |
+
libsndfile1 ffmpeg git \
|
| 8 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 9 |
+
|
| 10 |
+
# Python deps — CPU-only torch (smaller image, faster build)
|
| 11 |
+
RUN pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cpu
|
| 12 |
+
COPY requirements.txt .
|
| 13 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 14 |
+
RUN pip install --no-cache-dir 'datasets<4' librosa soundfile huggingface_hub
|
| 15 |
+
|
| 16 |
+
# Copy project
|
| 17 |
+
COPY src/ src/
|
| 18 |
+
COPY app.py .
|
| 19 |
+
COPY recipes.yaml .
|
| 20 |
+
|
| 21 |
+
# Cache 1750h recipe and upload to HF Dataset repo
|
| 22 |
+
# HF_TOKEN must be set as a Space secret for upload
|
| 23 |
+
CMD ["python", "-u", "app.py", "cache", \
|
| 24 |
+
"--recipe", "1750h", \
|
| 25 |
+
"--cache-dir", "mel_cache", \
|
| 26 |
+
"--repo", "tisaconundrum/grassmann-mel-cache"]
|
README.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Grassmann Cache
|
| 3 |
+
emoji: 📦
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: false
|
| 8 |
+
---
|
| 9 |
+
|
| 10 |
+
Caches 1750h of mel spectrograms for Grassmann ASR training.
|
app.py
ADDED
|
@@ -0,0 +1,518 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Grassmann Transcription — Unified CLI
|
| 4 |
+
|
| 5 |
+
One entry point for training, caching, evaluation, export, and inference.
|
| 6 |
+
Based on "Attention Is Not What You Need" (arXiv:2512.19428).
|
| 7 |
+
|
| 8 |
+
Usage:
|
| 9 |
+
python app.py train --from-cache tisaconundrum/grassmann-mel-cache
|
| 10 |
+
python app.py train --recipe 1750h
|
| 11 |
+
python app.py cache --recipe 1750h --repo tisaconundrum/grassmann-mel-cache
|
| 12 |
+
python app.py export --model checkpoints/bundle --output onnx_export/
|
| 13 |
+
python app.py eval --model eval_model/best_model.pt --config eval_model/config.json
|
| 14 |
+
python app.py transcribe --model checkpoints/bundle --audio speech.wav
|
| 15 |
+
python app.py demo
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import argparse
|
| 19 |
+
import os
|
| 20 |
+
import sys
|
| 21 |
+
import time
|
| 22 |
+
import threading
|
| 23 |
+
|
| 24 |
+
import torch
|
| 25 |
+
|
| 26 |
+
os.environ["PYTHONUNBUFFERED"] = "1"
|
| 27 |
+
sys.stdout.reconfigure(line_buffering=True)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# ── Shared utilities ─────────────────────────────────────────────────────
|
| 31 |
+
|
| 32 |
+
def _start_health_server(port: int = 7860) -> None:
|
| 33 |
+
from http.server import HTTPServer, BaseHTTPRequestHandler
|
| 34 |
+
|
| 35 |
+
class Handler(BaseHTTPRequestHandler):
|
| 36 |
+
def do_GET(self):
|
| 37 |
+
self.send_response(200)
|
| 38 |
+
self.send_header("Content-Type", "text/plain")
|
| 39 |
+
self.end_headers()
|
| 40 |
+
self.wfile.write(b"Running...")
|
| 41 |
+
|
| 42 |
+
def log_message(self, format, *args):
|
| 43 |
+
pass
|
| 44 |
+
|
| 45 |
+
server = HTTPServer(("0.0.0.0", port), Handler)
|
| 46 |
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
| 47 |
+
print(f" Health server on port {port}", flush=True)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _log_memory(label: str) -> None:
|
| 51 |
+
try:
|
| 52 |
+
import resource
|
| 53 |
+
rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
| 54 |
+
rss_mb = rss_kb / (1024 * 1024 if sys.platform == "darwin" else 1024)
|
| 55 |
+
print(f" [mem] {label}: RSS={rss_mb:.0f}MB", end="", flush=True)
|
| 56 |
+
except Exception:
|
| 57 |
+
print(f" [mem] {label}: RSS=unknown", end="", flush=True)
|
| 58 |
+
if torch.cuda.is_available():
|
| 59 |
+
alloc = torch.cuda.memory_allocated() / 1e6
|
| 60 |
+
reserved = torch.cuda.memory_reserved() / 1e6
|
| 61 |
+
print(f" GPU_alloc={alloc:.0f}MB GPU_reserved={reserved:.0f}MB", flush=True)
|
| 62 |
+
else:
|
| 63 |
+
print(flush=True)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _add_model_args(p: argparse.ArgumentParser) -> None:
|
| 67 |
+
p.add_argument("--layers", type=int, default=6)
|
| 68 |
+
p.add_argument("--d-model", type=int, default=256)
|
| 69 |
+
p.add_argument("--r-reduced", type=int, default=32)
|
| 70 |
+
p.add_argument("--lr", type=float, default=3e-4)
|
| 71 |
+
p.add_argument("--decoder-layers", type=int, default=4)
|
| 72 |
+
p.add_argument("--cross-window", type=int, default=8)
|
| 73 |
+
p.add_argument("--ctc", action="store_true", help="CTC-only mode")
|
| 74 |
+
p.add_argument("--conformer", action="store_true", help="Conformer blocks")
|
| 75 |
+
p.add_argument("--two-sided", action="store_true", help="Bidirectional mixing")
|
| 76 |
+
p.add_argument("--conv-kernel", type=int, default=31)
|
| 77 |
+
p.add_argument("--ablation", action="store_true", help="Symmetric-product ablation (no Grassmann geometry)")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _auto_pause_space() -> None:
|
| 81 |
+
try:
|
| 82 |
+
from huggingface_hub import HfApi
|
| 83 |
+
space_id = os.environ.get("SPACE_ID")
|
| 84 |
+
if space_id:
|
| 85 |
+
HfApi().pause_space(space_id)
|
| 86 |
+
print(f" Space {space_id} paused.", flush=True)
|
| 87 |
+
except Exception as e:
|
| 88 |
+
print(f" Note: could not auto-pause: {e}", flush=True)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
# ── train ────────────────────────────────────────────────────────────────
|
| 92 |
+
|
| 93 |
+
def cmd_train(args: argparse.Namespace) -> None:
|
| 94 |
+
import gc
|
| 95 |
+
from src.models.config import GrassmannConfig
|
| 96 |
+
from src.controllers.model_controller import ModelController
|
| 97 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 98 |
+
from src.repositories.dataset_repository import DatasetRepository
|
| 99 |
+
from src.repositories.cache_repository import CacheRepository
|
| 100 |
+
from src.services.dataset_service import DatasetService
|
| 101 |
+
|
| 102 |
+
_start_health_server()
|
| 103 |
+
|
| 104 |
+
config = GrassmannConfig(
|
| 105 |
+
epochs=args.epochs, batch_size=args.batch_size,
|
| 106 |
+
n_layers=args.layers, n_decoder_layers=args.decoder_layers,
|
| 107 |
+
cross_window=args.cross_window, d_model=args.d_model,
|
| 108 |
+
r_reduced=args.r_reduced, learning_rate=args.lr,
|
| 109 |
+
use_seq2seq=not args.ctc, use_conformer=args.conformer,
|
| 110 |
+
two_sided=args.two_sided, conv_kernel_size=args.conv_kernel,
|
| 111 |
+
use_ablation=args.ablation,
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
print("\n" + "=" * 60)
|
| 115 |
+
print(" Grassmann Transcription — Training")
|
| 116 |
+
print("=" * 60)
|
| 117 |
+
if torch.cuda.is_available():
|
| 118 |
+
gpu = torch.cuda.get_device_name(0)
|
| 119 |
+
mem = torch.cuda.get_device_properties(0).total_memory / 1e9
|
| 120 |
+
print(f" GPU: {gpu} ({mem:.1f} GB)")
|
| 121 |
+
else:
|
| 122 |
+
print(" GPU: None (CPU)")
|
| 123 |
+
mode = "CTC" if args.ctc else "Seq2Seq"
|
| 124 |
+
block = "Conformer" if args.conformer else "Plain"
|
| 125 |
+
mix = "two-sided" if args.two_sided else "causal"
|
| 126 |
+
mixer = "Ablation (symmetric)" if args.ablation else "Grassmann (Plücker)"
|
| 127 |
+
print(f" Mode: {mode} | Blocks: {block} ({mix})")
|
| 128 |
+
print(f" Mixer: {mixer}")
|
| 129 |
+
print(f" Encoder: {args.layers}L / d={args.d_model} / r={args.r_reduced}")
|
| 130 |
+
print("=" * 60 + "\n")
|
| 131 |
+
|
| 132 |
+
tokenizer = CharTokenizer()
|
| 133 |
+
cache_dir = os.path.join(args.save_dir, "mel_cache")
|
| 134 |
+
repo = DatasetRepository(config, tokenizer)
|
| 135 |
+
cache = CacheRepository(cache_dir)
|
| 136 |
+
dataset = DatasetService(repo, cache)
|
| 137 |
+
|
| 138 |
+
if args.from_cache:
|
| 139 |
+
dataset.load_from_hub_cache(args.from_cache).clean().preprocess().split(
|
| 140 |
+
max_samples=args.max_samples)
|
| 141 |
+
elif args.recipe:
|
| 142 |
+
dataset.load_recipe(args.recipe,
|
| 143 |
+
max_samples_per_source=args.max_per_source).clean().preprocess().split()
|
| 144 |
+
else:
|
| 145 |
+
dataset.load_dataset(args.dataset, split=args.split,
|
| 146 |
+
max_samples=args.max_samples).clean().preprocess().split()
|
| 147 |
+
|
| 148 |
+
gc.collect()
|
| 149 |
+
if torch.cuda.is_available():
|
| 150 |
+
torch.cuda.empty_cache()
|
| 151 |
+
_log_memory("After dataset load")
|
| 152 |
+
|
| 153 |
+
model_ctrl = ModelController(config, tokenizer)
|
| 154 |
+
model_ctrl.build()
|
| 155 |
+
assert model_ctrl.trainer is not None
|
| 156 |
+
model_ctrl.trainer.set_cosine_schedule(args.epochs, warmup_epochs=min(3, args.epochs // 5))
|
| 157 |
+
|
| 158 |
+
train_loader = dataset.get_train_loader(config.batch_size)
|
| 159 |
+
val_loader = dataset.get_val_loader(config.batch_size)
|
| 160 |
+
|
| 161 |
+
os.makedirs(args.save_dir, exist_ok=True)
|
| 162 |
+
best_val_loss = float("inf")
|
| 163 |
+
start_time = time.time()
|
| 164 |
+
_log_memory("Before training")
|
| 165 |
+
|
| 166 |
+
for epoch in range(1, args.epochs + 1):
|
| 167 |
+
epoch_start = time.time()
|
| 168 |
+
train_loss = model_ctrl.trainer.train_epoch(train_loader)
|
| 169 |
+
val_loss = model_ctrl.trainer.validate(val_loader)
|
| 170 |
+
|
| 171 |
+
gc.collect()
|
| 172 |
+
if torch.cuda.is_available():
|
| 173 |
+
torch.cuda.empty_cache()
|
| 174 |
+
|
| 175 |
+
if model_ctrl.trainer.scheduler is not None:
|
| 176 |
+
model_ctrl.trainer.scheduler.step()
|
| 177 |
+
|
| 178 |
+
lr = model_ctrl.trainer.optimizer.param_groups[0]["lr"]
|
| 179 |
+
elapsed = time.time() - epoch_start
|
| 180 |
+
total_elapsed = time.time() - start_time
|
| 181 |
+
mem_str = ""
|
| 182 |
+
if torch.cuda.is_available():
|
| 183 |
+
alloc = torch.cuda.memory_allocated() / 1e9
|
| 184 |
+
reserved = torch.cuda.memory_reserved() / 1e9
|
| 185 |
+
mem_str = f" GPU: {alloc:.1f}/{reserved:.1f}GB"
|
| 186 |
+
|
| 187 |
+
print(f" Epoch {epoch:3d}/{args.epochs} | "
|
| 188 |
+
f"train: {train_loss:.4f} val: {val_loss:.4f} "
|
| 189 |
+
f"lr: {lr:.2e} ({elapsed:.1f}s, total: {total_elapsed/60:.1f}min)"
|
| 190 |
+
f"{mem_str}", flush=True)
|
| 191 |
+
|
| 192 |
+
if val_loss < best_val_loss:
|
| 193 |
+
best_val_loss = val_loss
|
| 194 |
+
assert model_ctrl.nn_model is not None
|
| 195 |
+
torch.save(model_ctrl.nn_model.state_dict(),
|
| 196 |
+
os.path.join(args.save_dir, "best_model.pt"))
|
| 197 |
+
print(f" -> Saved best model (val_loss={val_loss:.4f})", flush=True)
|
| 198 |
+
|
| 199 |
+
if epoch % 10 == 0:
|
| 200 |
+
assert model_ctrl.nn_model is not None
|
| 201 |
+
torch.save(model_ctrl.nn_model.state_dict(),
|
| 202 |
+
os.path.join(args.save_dir, f"epoch_{epoch}.pt"))
|
| 203 |
+
|
| 204 |
+
if epoch % args.decode_every == 0 or epoch == 1:
|
| 205 |
+
_decode_samples(model_ctrl, dataset)
|
| 206 |
+
|
| 207 |
+
assert model_ctrl.nn_model is not None
|
| 208 |
+
torch.save(model_ctrl.nn_model.state_dict(), os.path.join(args.save_dir, "final_model.pt"))
|
| 209 |
+
|
| 210 |
+
test_loader = dataset.get_test_loader(config.batch_size)
|
| 211 |
+
test_loss = model_ctrl.trainer.validate(test_loader)
|
| 212 |
+
total_time = time.time() - start_time
|
| 213 |
+
|
| 214 |
+
print(f"\n{'=' * 60}")
|
| 215 |
+
print(f" Training Complete!")
|
| 216 |
+
print(f" Best val loss: {best_val_loss:.4f}")
|
| 217 |
+
print(f" Test loss: {test_loss:.4f}")
|
| 218 |
+
print(f" Total time: {total_time/60:.1f} minutes")
|
| 219 |
+
print(f"{'=' * 60}")
|
| 220 |
+
|
| 221 |
+
_decode_samples(model_ctrl, dataset, n=5)
|
| 222 |
+
_post_training(model_ctrl, config, args.save_dir, run_name=args.run_name)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def _decode_samples(model_ctrl, dataset, n: int = 3) -> None:
|
| 226 |
+
assert model_ctrl.trainer is not None and dataset._val_ds is not None
|
| 227 |
+
print("\n Sample transcriptions:", flush=True)
|
| 228 |
+
for i in range(min(n, len(dataset._val_ds))):
|
| 229 |
+
mel, tokens = dataset._val_ds[i]
|
| 230 |
+
pred_text = model_ctrl.predict(mel)
|
| 231 |
+
gt_text = model_ctrl.tokenizer.decode(tokens)
|
| 232 |
+
print(f" [{i}] GT: '{gt_text}'", flush=True)
|
| 233 |
+
print(f" [{i}] PRED: '{pred_text}'")
|
| 234 |
+
print()
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _post_training(model_ctrl, config, save_dir: str, run_name: str | None = None) -> None:
|
| 238 |
+
from src.services.export_service import export_seq2seq, export_ctc
|
| 239 |
+
|
| 240 |
+
def _prefixed(name: str) -> str:
|
| 241 |
+
"""Prefix a filename with run_name so uploads are distinguishable."""
|
| 242 |
+
if not run_name:
|
| 243 |
+
return name
|
| 244 |
+
base, ext = os.path.splitext(name)
|
| 245 |
+
return f"{run_name}-{base}{ext}"
|
| 246 |
+
|
| 247 |
+
try:
|
| 248 |
+
from src.services.model_hub_service import save_model
|
| 249 |
+
assert model_ctrl.nn_model is not None
|
| 250 |
+
save_model(model_ctrl.nn_model, config, os.path.join(save_dir, "bundle"))
|
| 251 |
+
print(" Model bundle saved.", flush=True)
|
| 252 |
+
except Exception as e:
|
| 253 |
+
print(f" Warning: bundle save failed: {e}", flush=True)
|
| 254 |
+
|
| 255 |
+
try:
|
| 256 |
+
onnx_dir = os.path.join(save_dir, "onnx")
|
| 257 |
+
if config.use_seq2seq:
|
| 258 |
+
export_seq2seq(model_ctrl.nn_model, config, onnx_dir)
|
| 259 |
+
else:
|
| 260 |
+
export_ctc(model_ctrl.nn_model, config, onnx_dir)
|
| 261 |
+
print(f" ONNX exported to {onnx_dir}/", flush=True)
|
| 262 |
+
except Exception as e:
|
| 263 |
+
print(f" Warning: ONNX export failed: {e}", flush=True)
|
| 264 |
+
|
| 265 |
+
# Rename artifacts with run_name prefix before upload
|
| 266 |
+
if run_name:
|
| 267 |
+
import shutil
|
| 268 |
+
# Rename best_model.pt → ablation-smol-best_model.pt etc.
|
| 269 |
+
for name in ["best_model.pt", "final_model.pt"]:
|
| 270 |
+
src = os.path.join(save_dir, name)
|
| 271 |
+
dst = os.path.join(save_dir, _prefixed(name))
|
| 272 |
+
if os.path.isfile(src):
|
| 273 |
+
shutil.move(src, dst)
|
| 274 |
+
# Rename ONNX files (model.onnx → ablation-smol-model.onnx etc.)
|
| 275 |
+
onnx_dir = os.path.join(save_dir, "onnx")
|
| 276 |
+
if os.path.isdir(onnx_dir):
|
| 277 |
+
for fname in list(os.listdir(onnx_dir)):
|
| 278 |
+
src = os.path.join(onnx_dir, fname)
|
| 279 |
+
dst = os.path.join(onnx_dir, _prefixed(fname))
|
| 280 |
+
if os.path.isfile(src):
|
| 281 |
+
shutil.move(src, dst)
|
| 282 |
+
# Rename bundle config files
|
| 283 |
+
bundle_dir = os.path.join(save_dir, "bundle")
|
| 284 |
+
if os.path.isdir(bundle_dir):
|
| 285 |
+
for fname in list(os.listdir(bundle_dir)):
|
| 286 |
+
src = os.path.join(bundle_dir, fname)
|
| 287 |
+
dst = os.path.join(bundle_dir, _prefixed(fname))
|
| 288 |
+
if os.path.isfile(src):
|
| 289 |
+
shutil.move(src, dst)
|
| 290 |
+
|
| 291 |
+
try:
|
| 292 |
+
from src.services.upload_service import upload_file, upload_directory
|
| 293 |
+
bundle_dir = os.path.join(save_dir, "bundle")
|
| 294 |
+
onnx_dir = os.path.join(save_dir, "onnx")
|
| 295 |
+
if os.path.isdir(bundle_dir):
|
| 296 |
+
for fname in sorted(os.listdir(bundle_dir)):
|
| 297 |
+
fpath = os.path.join(bundle_dir, fname)
|
| 298 |
+
if os.path.isfile(fpath):
|
| 299 |
+
upload_file(fpath)
|
| 300 |
+
if os.path.isdir(onnx_dir):
|
| 301 |
+
urls = upload_directory(onnx_dir)
|
| 302 |
+
print(f" Uploaded {len(urls)} ONNX files", flush=True)
|
| 303 |
+
best_name = _prefixed("best_model.pt")
|
| 304 |
+
best_path = os.path.join(save_dir, best_name)
|
| 305 |
+
if os.path.isfile(best_path):
|
| 306 |
+
upload_file(best_path)
|
| 307 |
+
except Exception as e:
|
| 308 |
+
print(f" Warning: upload failed: {e}", flush=True)
|
| 309 |
+
|
| 310 |
+
_auto_pause_space()
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
# ── cache ────────────────────────────────────────────────────────────────
|
| 314 |
+
|
| 315 |
+
def cmd_cache(args: argparse.Namespace) -> None:
|
| 316 |
+
from src.models.config import GrassmannConfig
|
| 317 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 318 |
+
from src.repositories.dataset_repository import DatasetRepository
|
| 319 |
+
from src.repositories.cache_repository import CacheRepository
|
| 320 |
+
from src.services.dataset_service import DatasetService
|
| 321 |
+
|
| 322 |
+
_start_health_server()
|
| 323 |
+
|
| 324 |
+
print("\n" + "=" * 60)
|
| 325 |
+
print(" Grassmann — Dataset Caching (CPU)")
|
| 326 |
+
print("=" * 60)
|
| 327 |
+
print(f" Recipe: {args.recipe}")
|
| 328 |
+
print(f" Upload to: {args.repo}")
|
| 329 |
+
print("=" * 60 + "\n")
|
| 330 |
+
|
| 331 |
+
config = GrassmannConfig()
|
| 332 |
+
tokenizer = CharTokenizer()
|
| 333 |
+
repo = DatasetRepository(config, tokenizer)
|
| 334 |
+
cache = CacheRepository(args.cache_dir)
|
| 335 |
+
dataset = DatasetService(repo, cache)
|
| 336 |
+
|
| 337 |
+
start = time.time()
|
| 338 |
+
dataset.load_recipe(args.recipe,
|
| 339 |
+
max_samples_per_source=args.max_per_source,
|
| 340 |
+
upload_repo=args.repo,
|
| 341 |
+
workers=args.workers)
|
| 342 |
+
print(f"\n Caching complete in {(time.time() - start)/60:.1f} minutes")
|
| 343 |
+
|
| 344 |
+
_auto_pause_space()
|
| 345 |
+
print(f"\n Done! Now train with: python app.py train --from-cache {args.repo}")
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
# ── export ───────────────────────────────────────────────────────────────
|
| 349 |
+
|
| 350 |
+
def cmd_export(args: argparse.Namespace) -> None:
|
| 351 |
+
import json
|
| 352 |
+
from dataclasses import asdict
|
| 353 |
+
from pathlib import Path
|
| 354 |
+
from src.services.model_hub_service import load_model
|
| 355 |
+
from src.services.export_service import export_seq2seq, export_ctc, verify_onnx
|
| 356 |
+
|
| 357 |
+
print(f"\n Loading model from {args.model}...")
|
| 358 |
+
model, config, tokenizer = load_model(args.model, device="cpu")
|
| 359 |
+
print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
|
| 360 |
+
print(f" Mode: {'seq2seq' if config.use_seq2seq else 'CTC'}")
|
| 361 |
+
|
| 362 |
+
if config.use_seq2seq:
|
| 363 |
+
export_seq2seq(model, config, args.output, args.opset)
|
| 364 |
+
else:
|
| 365 |
+
export_ctc(model, config, args.output, args.opset)
|
| 366 |
+
|
| 367 |
+
out = Path(args.output)
|
| 368 |
+
with open(out / "config.json", "w") as f:
|
| 369 |
+
json.dump(asdict(config), f, indent=2)
|
| 370 |
+
with open(out / "tokenizer_info.json", "w") as f:
|
| 371 |
+
json.dump({"type": "char", "vocab_size": tokenizer.vocab_size}, f, indent=2)
|
| 372 |
+
|
| 373 |
+
print(f"\n Export complete!")
|
| 374 |
+
if args.verify:
|
| 375 |
+
verify_onnx(args.output, config, config.use_seq2seq)
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
# ── eval ─────────────────────────────────────────────────────────────────
|
| 379 |
+
|
| 380 |
+
def cmd_eval(args: argparse.Namespace) -> None:
|
| 381 |
+
import json
|
| 382 |
+
from src.models.config import GrassmannConfig
|
| 383 |
+
from src.models.grassmann_model import GrassmannTranscriptionModel
|
| 384 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 385 |
+
from src.services.eval_service import evaluate_wer
|
| 386 |
+
|
| 387 |
+
print(" Loading model...", flush=True)
|
| 388 |
+
with open(args.config) as f:
|
| 389 |
+
config_dict = json.load(f)
|
| 390 |
+
config = GrassmannConfig(**{
|
| 391 |
+
k: v for k, v in config_dict.items()
|
| 392 |
+
if k in GrassmannConfig.__dataclass_fields__
|
| 393 |
+
})
|
| 394 |
+
tokenizer = CharTokenizer()
|
| 395 |
+
config.vocab_size = tokenizer.vocab_size
|
| 396 |
+
|
| 397 |
+
model = GrassmannTranscriptionModel(config)
|
| 398 |
+
model.load_state_dict(torch.load(args.model, map_location="cpu", weights_only=True))
|
| 399 |
+
model.eval()
|
| 400 |
+
print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
|
| 401 |
+
|
| 402 |
+
evaluate_wer(model, config, tokenizer, split=args.split,
|
| 403 |
+
max_samples=args.max_samples, show_errors=args.show_errors)
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
# ── transcribe ───────────────────────────────────────────────────────────
|
| 407 |
+
|
| 408 |
+
def cmd_transcribe(args: argparse.Namespace) -> None:
|
| 409 |
+
from src.models.config import GrassmannConfig
|
| 410 |
+
from src.controllers.pipeline_controller import PipelineController
|
| 411 |
+
|
| 412 |
+
config = GrassmannConfig()
|
| 413 |
+
pipeline = PipelineController(config)
|
| 414 |
+
pipeline.load_model(args.model)
|
| 415 |
+
text = pipeline.transcribe(args.audio)
|
| 416 |
+
print(f"\n {text}\n")
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
# ── demo ─────────────────────────────────────────────────────────────────
|
| 420 |
+
|
| 421 |
+
def cmd_demo() -> None:
|
| 422 |
+
from src.models.config import GrassmannConfig
|
| 423 |
+
from src.controllers.model_controller import ModelController
|
| 424 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 425 |
+
|
| 426 |
+
print("\n Running demo with synthetic data...\n")
|
| 427 |
+
config = GrassmannConfig(n_layers=2, d_model=128, r_reduced=16, d_ff=512,
|
| 428 |
+
n_decoder_layers=2, epochs=3, batch_size=4)
|
| 429 |
+
tokenizer = CharTokenizer()
|
| 430 |
+
config.vocab_size = tokenizer.vocab_size
|
| 431 |
+
|
| 432 |
+
model = ModelController(config, tokenizer)
|
| 433 |
+
model.build()
|
| 434 |
+
assert model.trainer is not None
|
| 435 |
+
device = model.trainer.device
|
| 436 |
+
mels = torch.randn(4, config.n_mels, 200, device=device)
|
| 437 |
+
|
| 438 |
+
assert model.nn_model is not None
|
| 439 |
+
if config.use_seq2seq:
|
| 440 |
+
tokens = tokenizer.encode("hello world").unsqueeze(0).expand(4, -1)
|
| 441 |
+
logits = model.nn_model(mels, tokens)
|
| 442 |
+
print(f" Forward pass OK — output: {logits.shape}")
|
| 443 |
+
else:
|
| 444 |
+
log_probs = model.nn_model(mels)
|
| 445 |
+
print(f" Forward pass OK — output: {log_probs.shape}")
|
| 446 |
+
|
| 447 |
+
pred_tokens = model.trainer.greedy_decode(mels[0:1])
|
| 448 |
+
decoded = tokenizer.decode(pred_tokens)
|
| 449 |
+
print(f" Greedy decode (untrained): '{decoded}'")
|
| 450 |
+
print("\n Demo complete!\n")
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
# ── CLI ──────────────────────────────────────────────────────────────────
|
| 454 |
+
|
| 455 |
+
def main() -> None:
|
| 456 |
+
parser = argparse.ArgumentParser(description="Grassmann Manifold Audio Transcription")
|
| 457 |
+
sub = parser.add_subparsers(dest="command")
|
| 458 |
+
|
| 459 |
+
# train
|
| 460 |
+
p = sub.add_parser("train", help="Train model (GPU)")
|
| 461 |
+
p.add_argument("--from-cache", default=None, help="HF Dataset repo with pre-cached mels")
|
| 462 |
+
p.add_argument("--recipe", default=None, help="Multi-dataset recipe (e.g. '1750h')")
|
| 463 |
+
p.add_argument("--dataset", default="openslr/librispeech_asr")
|
| 464 |
+
p.add_argument("--split", default="train.clean.100")
|
| 465 |
+
p.add_argument("--max-samples", type=int, default=None)
|
| 466 |
+
p.add_argument("--max-per-source", type=int, default=None)
|
| 467 |
+
p.add_argument("--epochs", type=int, default=50)
|
| 468 |
+
p.add_argument("--batch-size", type=int, default=32)
|
| 469 |
+
p.add_argument("--save-dir", default="checkpoints")
|
| 470 |
+
p.add_argument("--run-name", default=None, help="Prefix for uploaded artifacts (e.g. 'ablation-smol')")
|
| 471 |
+
p.add_argument("--decode-every", type=int, default=5)
|
| 472 |
+
_add_model_args(p)
|
| 473 |
+
|
| 474 |
+
# cache
|
| 475 |
+
p = sub.add_parser("cache", help="Cache dataset mels to HF Hub (CPU)")
|
| 476 |
+
p.add_argument("--recipe", required=True)
|
| 477 |
+
p.add_argument("--repo", required=True, help="HF Dataset repo")
|
| 478 |
+
p.add_argument("--cache-dir", default="mel_cache")
|
| 479 |
+
p.add_argument("--max-per-source", type=int, default=None)
|
| 480 |
+
p.add_argument("--workers", type=int, default=0, help="Parallel mel workers (0=sequential, default)")
|
| 481 |
+
|
| 482 |
+
# export
|
| 483 |
+
p = sub.add_parser("export", help="Export model to ONNX")
|
| 484 |
+
p.add_argument("--model", required=True)
|
| 485 |
+
p.add_argument("--output", default="onnx_export")
|
| 486 |
+
p.add_argument("--opset", type=int, default=18)
|
| 487 |
+
p.add_argument("--verify", action="store_true")
|
| 488 |
+
|
| 489 |
+
# eval
|
| 490 |
+
p = sub.add_parser("eval", help="Evaluate WER/CER on LibriSpeech")
|
| 491 |
+
p.add_argument("--model", default="eval_model/best_model.pt")
|
| 492 |
+
p.add_argument("--config", default="eval_model/config.json")
|
| 493 |
+
p.add_argument("--split", default="test.clean")
|
| 494 |
+
p.add_argument("--max-samples", type=int, default=None)
|
| 495 |
+
p.add_argument("--show-errors", type=int, default=10)
|
| 496 |
+
|
| 497 |
+
# transcribe
|
| 498 |
+
p = sub.add_parser("transcribe", help="Transcribe audio file")
|
| 499 |
+
p.add_argument("--model", required=True)
|
| 500 |
+
p.add_argument("--audio", required=True)
|
| 501 |
+
|
| 502 |
+
# demo
|
| 503 |
+
sub.add_parser("demo", help="Sanity check with synthetic data")
|
| 504 |
+
|
| 505 |
+
args = parser.parse_args()
|
| 506 |
+
commands = {
|
| 507 |
+
"train": cmd_train, "cache": cmd_cache, "export": cmd_export,
|
| 508 |
+
"eval": cmd_eval, "transcribe": cmd_transcribe, "demo": lambda a: cmd_demo(),
|
| 509 |
+
}
|
| 510 |
+
if args.command in commands:
|
| 511 |
+
commands[args.command](args)
|
| 512 |
+
else:
|
| 513 |
+
parser.print_help()
|
| 514 |
+
sys.exit(1)
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
if __name__ == "__main__":
|
| 518 |
+
main()
|
recipes.yaml
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Dataset recipes for training.
|
| 2 |
+
# Each recipe is a list of HuggingFace dataset sources.
|
| 3 |
+
# Use with: python app.py train --recipe <name>
|
| 4 |
+
#
|
| 5 |
+
# Fields per source:
|
| 6 |
+
# name: HuggingFace dataset ID (required)
|
| 7 |
+
# split: Dataset split name (required)
|
| 8 |
+
# config: Dataset config/subset (optional)
|
| 9 |
+
# max_samples: Cap samples from this source (optional)
|
| 10 |
+
|
| 11 |
+
schemas:
|
| 12 |
+
openslr/librispeech_asr:
|
| 13 |
+
audio: audio
|
| 14 |
+
text: text
|
| 15 |
+
mozilla-foundation/common_voice:
|
| 16 |
+
audio: audio
|
| 17 |
+
text: sentence
|
| 18 |
+
google/fleurs:
|
| 19 |
+
audio: audio
|
| 20 |
+
text: transcription
|
| 21 |
+
speechcolab/gigaspeech:
|
| 22 |
+
audio: audio
|
| 23 |
+
text: text
|
| 24 |
+
facebook/voxpopuli:
|
| 25 |
+
audio: audio
|
| 26 |
+
text: normalized_text
|
| 27 |
+
MLCommons/peoples_speech:
|
| 28 |
+
audio: audio
|
| 29 |
+
text: text
|
| 30 |
+
espnet/yodas:
|
| 31 |
+
audio: audio
|
| 32 |
+
text: text
|
| 33 |
+
|
| 34 |
+
recipes:
|
| 35 |
+
librispeech-100:
|
| 36 |
+
- name: openslr/librispeech_asr
|
| 37 |
+
split: train.clean.100
|
| 38 |
+
|
| 39 |
+
librispeech-960:
|
| 40 |
+
- name: openslr/librispeech_asr
|
| 41 |
+
split: train.clean.100
|
| 42 |
+
- name: openslr/librispeech_asr
|
| 43 |
+
split: train.clean.360
|
| 44 |
+
- name: openslr/librispeech_asr
|
| 45 |
+
split: train.other.500
|
| 46 |
+
|
| 47 |
+
1750h:
|
| 48 |
+
- name: openslr/librispeech_asr
|
| 49 |
+
split: train.clean.100
|
| 50 |
+
- name: openslr/librispeech_asr
|
| 51 |
+
split: train.clean.360
|
| 52 |
+
- name: openslr/librispeech_asr
|
| 53 |
+
split: train.other.500
|
| 54 |
+
- name: facebook/voxpopuli
|
| 55 |
+
config: en
|
| 56 |
+
split: train
|
| 57 |
+
- name: MLCommons/peoples_speech
|
| 58 |
+
config: clean
|
| 59 |
+
split: train
|
| 60 |
+
max_samples: 30000 # ~250h subset of 30K hours
|
requirements.txt
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.0
|
| 2 |
+
torchaudio>=2.0
|
| 3 |
+
datasets>=3.0
|
| 4 |
+
soundfile>=0.12
|
| 5 |
+
torchcodec>=0.1
|
| 6 |
+
sqids>=0.4
|
| 7 |
+
pyyaml>=6.0
|
src/__init__.py
ADDED
|
File without changes
|
src/controllers/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .model_controller import ModelController
|
| 2 |
+
from .pipeline_controller import PipelineController
|
src/controllers/model_controller.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ModelController: fluent API for model lifecycle.
|
| 3 |
+
|
| 4 |
+
Provides the chainable interface:
|
| 5 |
+
_model.build().train(...).test(...).save(...)
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from typing import Any, Self
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
from src.interfaces import IModel
|
| 15 |
+
from src.models.config import GrassmannConfig
|
| 16 |
+
from src.models.grassmann_model import GrassmannTranscriptionModel
|
| 17 |
+
from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
|
| 18 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 19 |
+
from src.services.training_service import TrainingService
|
| 20 |
+
from src.services.seq2seq_training_service import Seq2SeqTrainingService
|
| 21 |
+
from src.views.console_view import ConsoleView
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class ModelController(IModel):
|
| 25 |
+
"""Orchestrates model building, training, evaluation, and inference."""
|
| 26 |
+
|
| 27 |
+
def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer):
|
| 28 |
+
self.config = config
|
| 29 |
+
self.tokenizer = tokenizer
|
| 30 |
+
self.view = ConsoleView()
|
| 31 |
+
self.nn_model: GrassmannTranscriptionModel | GrassmannSeq2SeqModel | None = None
|
| 32 |
+
self.trainer: TrainingService | Seq2SeqTrainingService | None = None
|
| 33 |
+
|
| 34 |
+
def build(self) -> Self:
|
| 35 |
+
self.config.vocab_size = self.tokenizer.vocab_size
|
| 36 |
+
|
| 37 |
+
if self.config.use_seq2seq:
|
| 38 |
+
self.nn_model = GrassmannSeq2SeqModel(self.config)
|
| 39 |
+
self.trainer = Seq2SeqTrainingService(self.nn_model, self.config)
|
| 40 |
+
else:
|
| 41 |
+
self.nn_model = GrassmannTranscriptionModel(self.config)
|
| 42 |
+
self.trainer = TrainingService(self.nn_model, self.config)
|
| 43 |
+
|
| 44 |
+
param_count = sum(p.numel() for p in self.nn_model.parameters())
|
| 45 |
+
self.view.show_model_info(self.config, param_count)
|
| 46 |
+
return self
|
| 47 |
+
|
| 48 |
+
def train(self, train_loader: Any, val_loader: Any, epochs: int | None = None) -> Self:
|
| 49 |
+
assert self.trainer is not None, "Call build() first"
|
| 50 |
+
epochs = epochs or self.config.epochs
|
| 51 |
+
|
| 52 |
+
history = self.trainer.train_loop(
|
| 53 |
+
train_loader, val_loader, epochs, callback=self.view
|
| 54 |
+
)
|
| 55 |
+
self.view.show_training_complete(history)
|
| 56 |
+
return self
|
| 57 |
+
|
| 58 |
+
def test(self, test_loader: Any) -> dict:
|
| 59 |
+
assert self.trainer is not None, "Call build() and train() first"
|
| 60 |
+
test_loss = self.trainer.validate(test_loader)
|
| 61 |
+
results = {"test_loss": test_loss}
|
| 62 |
+
self.view.show_test_results(results)
|
| 63 |
+
return results
|
| 64 |
+
|
| 65 |
+
def fit(self, dataset: Any) -> Self:
|
| 66 |
+
"""Convenience: build + train + validate."""
|
| 67 |
+
self.build()
|
| 68 |
+
|
| 69 |
+
train_loader = dataset.get_train_loader(self.config.batch_size)
|
| 70 |
+
val_loader = dataset.get_val_loader(self.config.batch_size)
|
| 71 |
+
|
| 72 |
+
self.train(train_loader, val_loader)
|
| 73 |
+
return self
|
| 74 |
+
|
| 75 |
+
def predict(self, audio: torch.Tensor) -> str:
|
| 76 |
+
assert self.trainer is not None, "Call build() and train()/load() first"
|
| 77 |
+
pred_tokens = self.trainer.greedy_decode(audio)
|
| 78 |
+
text = self.tokenizer.decode(pred_tokens)
|
| 79 |
+
return text
|
| 80 |
+
|
| 81 |
+
def save(self, path: str) -> Self:
|
| 82 |
+
assert self.nn_model is not None
|
| 83 |
+
torch.save(self.nn_model.state_dict(), path)
|
| 84 |
+
self.view.show_message(f"Model saved to {path}")
|
| 85 |
+
return self
|
| 86 |
+
|
| 87 |
+
def load(self, path: str) -> Self:
|
| 88 |
+
if self.nn_model is None:
|
| 89 |
+
self.build()
|
| 90 |
+
assert self.nn_model is not None
|
| 91 |
+
self.nn_model.load_state_dict(torch.load(path, weights_only=True))
|
| 92 |
+
self.view.show_message(f"Model loaded from {path}")
|
| 93 |
+
return self
|
src/controllers/pipeline_controller.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PipelineController: end-to-end orchestration.
|
| 3 |
+
|
| 4 |
+
Wires together dataset, model, and services into a single fluent pipeline.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import Self
|
| 10 |
+
|
| 11 |
+
from src.interfaces import IDataset
|
| 12 |
+
from src.models.config import GrassmannConfig
|
| 13 |
+
from src.services.audio_service import AudioService
|
| 14 |
+
from src.services.dataset_service import DatasetService
|
| 15 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 16 |
+
from src.repositories.dataset_repository import DatasetRepository
|
| 17 |
+
from src.repositories.cache_repository import CacheRepository
|
| 18 |
+
from src.controllers.model_controller import ModelController
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class PipelineController:
|
| 22 |
+
"""
|
| 23 |
+
Top-level controller for the full transcription pipeline.
|
| 24 |
+
|
| 25 |
+
Usage (local files):
|
| 26 |
+
pipeline = PipelineController(config)
|
| 27 |
+
pipeline.prepare_dataset("path/to/data")
|
| 28 |
+
.build_model()
|
| 29 |
+
.train()
|
| 30 |
+
.test()
|
| 31 |
+
.transcribe("path/to/audio.wav")
|
| 32 |
+
|
| 33 |
+
Usage (HuggingFace):
|
| 34 |
+
pipeline = PipelineController(config)
|
| 35 |
+
pipeline.prepare_hf_dataset("openslr/librispeech_asr",
|
| 36 |
+
split="train.clean.100")
|
| 37 |
+
.build_model()
|
| 38 |
+
.train()
|
| 39 |
+
.test()
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
def __init__(self, config: GrassmannConfig | None = None,
|
| 43 |
+
cache_dir: str = "checkpoints/mel_cache"):
|
| 44 |
+
self.config = config or GrassmannConfig()
|
| 45 |
+
self.tokenizer = CharTokenizer()
|
| 46 |
+
self.audio_service = AudioService(self.config)
|
| 47 |
+
self._repo = DatasetRepository(self.config, self.tokenizer)
|
| 48 |
+
self._cache = CacheRepository(cache_dir)
|
| 49 |
+
self.dataset: DatasetService | None = None
|
| 50 |
+
self.model: ModelController | None = None
|
| 51 |
+
|
| 52 |
+
def prepare_hf_dataset(
|
| 53 |
+
self,
|
| 54 |
+
dataset_name: str,
|
| 55 |
+
config_name: str | None = None,
|
| 56 |
+
split: str | None = None,
|
| 57 |
+
max_samples: int | None = None,
|
| 58 |
+
) -> Self:
|
| 59 |
+
"""Load a dataset from HuggingFace Hub."""
|
| 60 |
+
self.dataset = DatasetService(self._repo, self._cache)
|
| 61 |
+
self.dataset.load_dataset(
|
| 62 |
+
dataset_name, split=split,
|
| 63 |
+
config_name=config_name,
|
| 64 |
+
max_samples=max_samples,
|
| 65 |
+
).clean().preprocess().split()
|
| 66 |
+
return self
|
| 67 |
+
|
| 68 |
+
def build_model(self) -> Self:
|
| 69 |
+
self.model = ModelController(self.config, self.tokenizer)
|
| 70 |
+
self.model.build()
|
| 71 |
+
return self
|
| 72 |
+
|
| 73 |
+
def train(self, epochs: int | None = None) -> Self:
|
| 74 |
+
assert self.model is not None and self.dataset is not None
|
| 75 |
+
train_loader = self.dataset.get_train_loader(self.config.batch_size)
|
| 76 |
+
val_loader = self.dataset.get_val_loader(self.config.batch_size)
|
| 77 |
+
self.model.train(train_loader, val_loader, epochs)
|
| 78 |
+
return self
|
| 79 |
+
|
| 80 |
+
def test(self) -> dict:
|
| 81 |
+
assert self.model is not None and self.dataset is not None
|
| 82 |
+
test_loader = self.dataset.get_test_loader(self.config.batch_size)
|
| 83 |
+
return self.model.test(test_loader)
|
| 84 |
+
|
| 85 |
+
def transcribe(self, audio_path: str) -> str:
|
| 86 |
+
assert self.model is not None
|
| 87 |
+
waveform = self.audio_service.load(audio_path)
|
| 88 |
+
mel = self.audio_service.extract_features(waveform)
|
| 89 |
+
mel = self.audio_service.normalize(mel)
|
| 90 |
+
return self.model.predict(mel)
|
| 91 |
+
|
| 92 |
+
def save_model(self, path: str) -> Self:
|
| 93 |
+
assert self.model is not None
|
| 94 |
+
self.model.save(path)
|
| 95 |
+
return self
|
| 96 |
+
|
| 97 |
+
def load_model(self, path: str) -> Self:
|
| 98 |
+
if self.model is None:
|
| 99 |
+
self.build_model()
|
| 100 |
+
assert self.model is not None
|
| 101 |
+
self.model.load(path)
|
| 102 |
+
return self
|
src/interfaces/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .i_dataset import IDataset
|
| 2 |
+
from .i_model import IModel
|
| 3 |
+
from .i_audio import IAudioProcessor
|
| 4 |
+
from .i_tokenizer import ITokenizer
|
src/interfaces/i_audio.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Any
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class IAudioProcessor(ABC):
|
| 8 |
+
"""Interface for audio feature extraction."""
|
| 9 |
+
|
| 10 |
+
@abstractmethod
|
| 11 |
+
def load(self, path: str) -> torch.Tensor:
|
| 12 |
+
"""Load raw audio from file. Returns waveform tensor."""
|
| 13 |
+
...
|
| 14 |
+
|
| 15 |
+
@abstractmethod
|
| 16 |
+
def extract_features(self, waveform: torch.Tensor) -> torch.Tensor:
|
| 17 |
+
"""Extract features (e.g. mel spectrogram) from waveform."""
|
| 18 |
+
...
|
| 19 |
+
|
| 20 |
+
@abstractmethod
|
| 21 |
+
def normalize(self, features: torch.Tensor) -> torch.Tensor:
|
| 22 |
+
"""Normalize features for model input."""
|
| 23 |
+
...
|
src/interfaces/i_dataset.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Self
|
| 3 |
+
|
| 4 |
+
from torch.utils.data import DataLoader
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class IDataset(ABC):
|
| 8 |
+
"""Interface for dataset loading and preprocessing."""
|
| 9 |
+
|
| 10 |
+
@abstractmethod
|
| 11 |
+
def load(self, path: str) -> Self:
|
| 12 |
+
"""Load dataset from path."""
|
| 13 |
+
...
|
| 14 |
+
|
| 15 |
+
@abstractmethod
|
| 16 |
+
def clean(self) -> Self:
|
| 17 |
+
"""Clean and filter the dataset."""
|
| 18 |
+
...
|
| 19 |
+
|
| 20 |
+
@abstractmethod
|
| 21 |
+
def preprocess(self) -> Self:
|
| 22 |
+
"""Preprocess dataset (extract features, tokenize labels)."""
|
| 23 |
+
...
|
| 24 |
+
|
| 25 |
+
@abstractmethod
|
| 26 |
+
def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1) -> Self:
|
| 27 |
+
"""Split into train/val/test sets."""
|
| 28 |
+
...
|
| 29 |
+
|
| 30 |
+
@abstractmethod
|
| 31 |
+
def get_train_loader(self, batch_size: int) -> DataLoader:
|
| 32 |
+
...
|
| 33 |
+
|
| 34 |
+
@abstractmethod
|
| 35 |
+
def get_val_loader(self, batch_size: int) -> DataLoader:
|
| 36 |
+
...
|
| 37 |
+
|
| 38 |
+
@abstractmethod
|
| 39 |
+
def get_test_loader(self, batch_size: int) -> DataLoader:
|
| 40 |
+
...
|
src/interfaces/i_model.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
from typing import Any, Self
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class IModel(ABC):
|
| 8 |
+
"""Interface for the trainable model."""
|
| 9 |
+
|
| 10 |
+
@abstractmethod
|
| 11 |
+
def build(self) -> Self:
|
| 12 |
+
"""Build / initialize the model architecture."""
|
| 13 |
+
...
|
| 14 |
+
|
| 15 |
+
@abstractmethod
|
| 16 |
+
def train(self, train_loader: Any, val_loader: Any, epochs: int) -> Self:
|
| 17 |
+
"""Train the model."""
|
| 18 |
+
...
|
| 19 |
+
|
| 20 |
+
@abstractmethod
|
| 21 |
+
def test(self, test_loader: Any) -> dict:
|
| 22 |
+
"""Evaluate on test set. Returns metrics dict."""
|
| 23 |
+
...
|
| 24 |
+
|
| 25 |
+
@abstractmethod
|
| 26 |
+
def fit(self, dataset: Any) -> Self:
|
| 27 |
+
"""Convenience: preprocess + train + validate in one call."""
|
| 28 |
+
...
|
| 29 |
+
|
| 30 |
+
@abstractmethod
|
| 31 |
+
def predict(self, audio: torch.Tensor) -> str:
|
| 32 |
+
"""Transcribe audio tensor to text."""
|
| 33 |
+
...
|
| 34 |
+
|
| 35 |
+
@abstractmethod
|
| 36 |
+
def save(self, path: str) -> Self:
|
| 37 |
+
"""Save model checkpoint."""
|
| 38 |
+
...
|
| 39 |
+
|
| 40 |
+
@abstractmethod
|
| 41 |
+
def load(self, path: str) -> Self:
|
| 42 |
+
"""Load model checkpoint."""
|
| 43 |
+
...
|
src/interfaces/i_tokenizer.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from abc import ABC, abstractmethod
|
| 2 |
+
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ITokenizer(ABC):
|
| 7 |
+
"""Interface for text tokenization (transcription labels)."""
|
| 8 |
+
|
| 9 |
+
@abstractmethod
|
| 10 |
+
def encode(self, text: str) -> torch.Tensor:
|
| 11 |
+
"""Encode text string to token indices."""
|
| 12 |
+
...
|
| 13 |
+
|
| 14 |
+
@abstractmethod
|
| 15 |
+
def decode(self, tokens: torch.Tensor) -> str:
|
| 16 |
+
"""Decode token indices back to text string."""
|
| 17 |
+
...
|
| 18 |
+
|
| 19 |
+
@property
|
| 20 |
+
@abstractmethod
|
| 21 |
+
def vocab_size(self) -> int:
|
| 22 |
+
...
|
| 23 |
+
|
| 24 |
+
@property
|
| 25 |
+
@abstractmethod
|
| 26 |
+
def blank_id(self) -> int:
|
| 27 |
+
"""CTC blank token id."""
|
| 28 |
+
...
|
src/models/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .config import GrassmannConfig
|
| 2 |
+
from .grassmann_layer import CausalGrassmannLayer, GrassmannMixer
|
| 3 |
+
from .ablation_layer import AblationLayer, AblationMixer
|
| 4 |
+
from .grassmann_model import GrassmannTranscriptionModel
|
| 5 |
+
from .grassmann_cross_layer import GrassmannCrossLayer
|
| 6 |
+
from .grassmann_decoder import GrassmannDecoder
|
| 7 |
+
from .grassmann_seq2seq import GrassmannSeq2SeqModel
|
| 8 |
+
from .conv_module import ConvModule
|
| 9 |
+
from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
|
src/models/ablation_layer.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Ablation Mixing Layer — Symmetric Product Baseline.
|
| 3 |
+
|
| 4 |
+
Drop-in replacement for the Grassmann/Plücker mixer that uses plain
|
| 5 |
+
element-wise symmetric products instead of antisymmetric Plücker
|
| 6 |
+
coordinates. Everything else (linear reduction, multi-scale offsets,
|
| 7 |
+
mean-first anchoring, gated fusion, FFN) stays identical so the only
|
| 8 |
+
variable is whether the Grassmann geometry matters.
|
| 9 |
+
|
| 10 |
+
Matches Dhruv's grassmann-speech implementation:
|
| 11 |
+
- symmetric_products: a_i * b_j + a_j * b_i (vs Plücker: a_i * b_j - a_j * b_i)
|
| 12 |
+
- symmetric_norm_sq: |a|²|b|² + (a·b)² - 2·Σ(aᵢ²bᵢ²) (vs Lagrange: |a|²|b|² - (a·b)²)
|
| 13 |
+
- flip = +1.0 in two-sided mode (vs Plücker: flip = -1.0)
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
import torch.nn as nn
|
| 18 |
+
|
| 19 |
+
from .config import GrassmannConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class SymmetricEncoder(nn.Module):
|
| 23 |
+
"""
|
| 24 |
+
Computes element-wise symmetric products for pairs of reduced vectors.
|
| 25 |
+
|
| 26 |
+
Same index pairs and output dimensionality as PluckerEncoder,
|
| 27 |
+
but uses p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
|
| 28 |
+
instead of the antisymmetric Plücker form.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
def __init__(self, r: int, eps: float = 1e-8):
|
| 32 |
+
super().__init__()
|
| 33 |
+
self.r = r
|
| 34 |
+
self.eps = eps
|
| 35 |
+
idx_i, idx_j = [], []
|
| 36 |
+
for i in range(r):
|
| 37 |
+
for j in range(i + 1, r):
|
| 38 |
+
idx_i.append(i)
|
| 39 |
+
idx_j.append(j)
|
| 40 |
+
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
|
| 41 |
+
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
|
| 42 |
+
|
| 43 |
+
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
|
| 44 |
+
"""
|
| 45 |
+
Compute normalized symmetric products.
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
z_t: (batch, seq, r)
|
| 49 |
+
z_delta: (batch, seq, r)
|
| 50 |
+
|
| 51 |
+
Returns:
|
| 52 |
+
(batch, seq, C(r,2))
|
| 53 |
+
"""
|
| 54 |
+
# Symmetric: p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
|
| 55 |
+
p = (
|
| 56 |
+
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
|
| 57 |
+
+ z_t[..., self.idx_j] * z_delta[..., self.idx_i]
|
| 58 |
+
)
|
| 59 |
+
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
|
| 60 |
+
return p / norm
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _symmetric_norm_sq(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
|
| 64 |
+
"""
|
| 65 |
+
Squared norm of symmetric products from dot products alone.
|
| 66 |
+
Matches grassmann-speech/plucker.py symmetric_norm_sq:
|
| 67 |
+
|a|²|b|² + (a·b)² - 2·Σ(aᵢ²·bᵢ²)
|
| 68 |
+
"""
|
| 69 |
+
aa = (a * a).sum(-1, keepdim=True)
|
| 70 |
+
bb = (b * b).sum(-1, keepdim=True)
|
| 71 |
+
ab = (a * b).sum(-1, keepdim=True)
|
| 72 |
+
return aa * bb + ab * ab - 2 * (a * a * b * b).sum(-1, keepdim=True)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class AblationMixer(nn.Module):
|
| 76 |
+
"""
|
| 77 |
+
Symmetric-product mixer — ablation baseline for GrassmannMixer.
|
| 78 |
+
|
| 79 |
+
Identical architecture (linear reduction → multi-scale pairing →
|
| 80 |
+
encoding → mean-first anchoring → projection) but with symmetric
|
| 81 |
+
products replacing Plücker coordinates.
|
| 82 |
+
|
| 83 |
+
Key differences from GrassmannMixer (matching grassmann-speech):
|
| 84 |
+
1. Symmetric products instead of Plücker coordinates
|
| 85 |
+
2. symmetric_norm_sq for anchored weighting
|
| 86 |
+
3. flip = +1.0 in two-sided mode (symmetric products are symmetric
|
| 87 |
+
under swap, so no sign flip — use positive accumulation)
|
| 88 |
+
"""
|
| 89 |
+
|
| 90 |
+
def __init__(self, d: int, r: int, offsets: list[int],
|
| 91 |
+
two_sided: bool = False, eps: float = 1e-8):
|
| 92 |
+
super().__init__()
|
| 93 |
+
self.offsets = offsets
|
| 94 |
+
self.two_sided = two_sided
|
| 95 |
+
self.eps = eps
|
| 96 |
+
plucker_dim = r * (r - 1) // 2 # same dimensionality
|
| 97 |
+
|
| 98 |
+
self.W_red = nn.Linear(d, r)
|
| 99 |
+
self.encoder = SymmetricEncoder(r, eps=eps)
|
| 100 |
+
self.W_plu = nn.Linear(plucker_dim, d)
|
| 101 |
+
|
| 102 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 103 |
+
"""Mean-first (anchored) form, matching grassmann-speech."""
|
| 104 |
+
B, L, D = h.shape
|
| 105 |
+
z = self.W_red(h)
|
| 106 |
+
|
| 107 |
+
partner = torch.zeros_like(z)
|
| 108 |
+
count = z.new_zeros(1, L, 1)
|
| 109 |
+
|
| 110 |
+
for delta in self.offsets:
|
| 111 |
+
if delta >= L:
|
| 112 |
+
continue
|
| 113 |
+
earlier = z[:, :L - delta, :]
|
| 114 |
+
later = z[:, delta:, :]
|
| 115 |
+
|
| 116 |
+
# Weight by symmetric norm (from dot products, no coordinate expansion)
|
| 117 |
+
w = _symmetric_norm_sq(earlier, later).clamp_min(0).sqrt().clamp_min(self.eps).reciprocal()
|
| 118 |
+
|
| 119 |
+
partner[:, delta:] += w * earlier
|
| 120 |
+
count[:, delta:] += 1
|
| 121 |
+
|
| 122 |
+
if self.two_sided:
|
| 123 |
+
# flip = +1.0 for symmetric (vs -1.0 for Plücker)
|
| 124 |
+
# Symmetric products are invariant under swap(a,b),
|
| 125 |
+
# so no sign flip needed to distinguish direction
|
| 126 |
+
partner[:, :L - delta] += w * later
|
| 127 |
+
count[:, :L - delta] += 1
|
| 128 |
+
|
| 129 |
+
count = count.clamp(min=1)
|
| 130 |
+
avg_partner = partner / count
|
| 131 |
+
p_final = self.encoder(z, avg_partner)
|
| 132 |
+
return self.W_plu(p_final)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
class AblationLayer(nn.Module):
|
| 136 |
+
"""
|
| 137 |
+
Drop-in replacement for CausalGrassmannLayer using symmetric products.
|
| 138 |
+
|
| 139 |
+
Same gated fusion + FFN, just swaps the mixer.
|
| 140 |
+
"""
|
| 141 |
+
|
| 142 |
+
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
|
| 143 |
+
super().__init__()
|
| 144 |
+
d = config.d_model
|
| 145 |
+
r = config.r_reduced
|
| 146 |
+
two_sided = getattr(config, "two_sided", False)
|
| 147 |
+
|
| 148 |
+
self.mixer = AblationMixer(
|
| 149 |
+
d, r, config.window_offsets,
|
| 150 |
+
two_sided=two_sided, eps=config.plucker_eps,
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
self.W_gate = nn.Linear(2 * d, d)
|
| 154 |
+
self.norm_mix = nn.LayerNorm(d)
|
| 155 |
+
self.dropout_mix = nn.Dropout(config.dropout)
|
| 156 |
+
|
| 157 |
+
self.ffn = nn.Sequential(
|
| 158 |
+
nn.Linear(d, config.d_ff),
|
| 159 |
+
nn.GELU(),
|
| 160 |
+
nn.Dropout(config.dropout),
|
| 161 |
+
nn.Linear(config.d_ff, d),
|
| 162 |
+
nn.Dropout(config.dropout),
|
| 163 |
+
)
|
| 164 |
+
self.norm_ffn = nn.LayerNorm(d)
|
| 165 |
+
|
| 166 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 167 |
+
g = self.mixer(h)
|
| 168 |
+
|
| 169 |
+
u = torch.cat([h, g], dim=-1)
|
| 170 |
+
alpha = torch.sigmoid(self.W_gate(u))
|
| 171 |
+
h_mix = h + (1 - alpha) * (g - h)
|
| 172 |
+
|
| 173 |
+
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
|
| 174 |
+
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
|
| 175 |
+
return h_out
|
src/models/config.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from dataclasses import dataclass, field
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
@dataclass
|
| 5 |
+
class GrassmannConfig:
|
| 6 |
+
"""Configuration for the Grassmann Transcription Model."""
|
| 7 |
+
|
| 8 |
+
# Audio features
|
| 9 |
+
n_mels: int = 80
|
| 10 |
+
sample_rate: int = 16000
|
| 11 |
+
n_fft: int = 400
|
| 12 |
+
hop_length: int = 160
|
| 13 |
+
|
| 14 |
+
# Model architecture
|
| 15 |
+
d_model: int = 256
|
| 16 |
+
r_reduced: int = 32
|
| 17 |
+
n_layers: int = 6
|
| 18 |
+
d_ff: int = 1024 # 4 * d_model
|
| 19 |
+
dropout: float = 0.1
|
| 20 |
+
|
| 21 |
+
# Decoder architecture
|
| 22 |
+
n_decoder_layers: int = 4
|
| 23 |
+
cross_window: int = 8 # encoder positions per side for cross-mixing
|
| 24 |
+
|
| 25 |
+
# Grassmann-specific
|
| 26 |
+
window_offsets: list[int] = field(default_factory=lambda: [1, 2, 4, 8, 12, 16])
|
| 27 |
+
plucker_eps: float = 1e-8
|
| 28 |
+
two_sided: bool = False # True for CTC/encoder (bidirectional)
|
| 29 |
+
|
| 30 |
+
# Conformer options
|
| 31 |
+
use_conformer: bool = False # True = Conformer blocks, False = plain Grassmann
|
| 32 |
+
conv_kernel_size: int = 31 # ConvModule kernel size
|
| 33 |
+
conv_channels: int = 256 # Subsampling conv channels
|
| 34 |
+
|
| 35 |
+
# Ablation: swap Plücker for symmetric products to test if geometry matters
|
| 36 |
+
use_ablation: bool = False
|
| 37 |
+
|
| 38 |
+
# Training
|
| 39 |
+
learning_rate: float = 3e-4
|
| 40 |
+
weight_decay: float = 1e-2
|
| 41 |
+
batch_size: int = 16
|
| 42 |
+
epochs: int = 30
|
| 43 |
+
max_seq_len: int = 2048 # LibriSpeech clips up to ~30s → ~750 frames after subsampling
|
| 44 |
+
|
| 45 |
+
# Architecture mode
|
| 46 |
+
use_seq2seq: bool = True # True = encoder-decoder, False = CTC-only
|
| 47 |
+
|
| 48 |
+
# Tokenizer
|
| 49 |
+
tokenizer_type: str = "char" # "char" or "bpe"
|
| 50 |
+
bpe_vocab_size: int = 1000 # BPE vocab size if tokenizer_type="bpe"
|
| 51 |
+
|
| 52 |
+
# Vocab (set dynamically by tokenizer)
|
| 53 |
+
vocab_size: int = 29 # 26 letters + space + apostrophe + blank
|
| 54 |
+
|
| 55 |
+
@property
|
| 56 |
+
def plucker_dim(self) -> int:
|
| 57 |
+
"""Dimension of Plücker coordinate vector: C(r, 2)."""
|
| 58 |
+
return self.r_reduced * (self.r_reduced - 1) // 2
|
src/models/conformer_block.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Conformer Block with Grassmann Mixer.
|
| 3 |
+
|
| 4 |
+
Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100)
|
| 5 |
+
with the self-attention slot replaced by a Grassmann mixer:
|
| 6 |
+
|
| 7 |
+
x → +0.5*FFN → +Mixer → +Conv → +0.5*FFN → LayerNorm
|
| 8 |
+
|
| 9 |
+
This gives the best of both worlds:
|
| 10 |
+
- Grassmann mixer for sequence-level subspace geometry
|
| 11 |
+
- ConvModule for local temporal pattern capture
|
| 12 |
+
- Macaron FFN sandwich for nonlinear capacity
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
|
| 18 |
+
from .config import GrassmannConfig
|
| 19 |
+
from .conv_module import ConvModule
|
| 20 |
+
from .grassmann_layer import GrassmannMixer
|
| 21 |
+
from .ablation_layer import AblationMixer
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential:
|
| 25 |
+
"""Conformer-style FFN: LayerNorm → Linear → SiLU → Dropout → Linear → Dropout."""
|
| 26 |
+
return nn.Sequential(
|
| 27 |
+
nn.LayerNorm(d),
|
| 28 |
+
nn.Linear(d, d_ff),
|
| 29 |
+
nn.SiLU(),
|
| 30 |
+
nn.Dropout(dropout),
|
| 31 |
+
nn.Linear(d_ff, d),
|
| 32 |
+
nn.Dropout(dropout),
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class GatedGrassmannMixer(nn.Module):
|
| 37 |
+
"""
|
| 38 |
+
Two-sided Grassmann mix with sigmoid gate as a residual branch.
|
| 39 |
+
(1 - α)(g - u) formulation from Dhruv's implementation.
|
| 40 |
+
"""
|
| 41 |
+
|
| 42 |
+
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
|
| 43 |
+
super().__init__()
|
| 44 |
+
self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps)
|
| 45 |
+
self.gate = nn.Linear(2 * d, d)
|
| 46 |
+
|
| 47 |
+
def forward(self, u: torch.Tensor) -> torch.Tensor:
|
| 48 |
+
g = self.mixer(u)
|
| 49 |
+
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
|
| 50 |
+
return (1 - alpha) * (g - u)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class GatedAblationMixer(nn.Module):
|
| 54 |
+
"""
|
| 55 |
+
Ablation baseline: symmetric-product mixer with the same gate.
|
| 56 |
+
Drop-in replacement for GatedGrassmannMixer.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
|
| 60 |
+
super().__init__()
|
| 61 |
+
self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps)
|
| 62 |
+
self.gate = nn.Linear(2 * d, d)
|
| 63 |
+
|
| 64 |
+
def forward(self, u: torch.Tensor) -> torch.Tensor:
|
| 65 |
+
g = self.mixer(u)
|
| 66 |
+
alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
|
| 67 |
+
return (1 - alpha) * (g - u)
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class GrassmannConformerBlock(nn.Module):
|
| 71 |
+
"""
|
| 72 |
+
Conformer block with Grassmann mixer replacing self-attention.
|
| 73 |
+
|
| 74 |
+
Architecture (Macaron block):
|
| 75 |
+
x → +0.5*FFN1 → +GatedGrassmannMixer → +ConvModule → +0.5*FFN2 → LayerNorm
|
| 76 |
+
"""
|
| 77 |
+
|
| 78 |
+
def __init__(self, config: GrassmannConfig):
|
| 79 |
+
super().__init__()
|
| 80 |
+
d = config.d_model
|
| 81 |
+
d_ff = config.d_ff
|
| 82 |
+
r = config.r_reduced
|
| 83 |
+
dropout = config.dropout
|
| 84 |
+
kernel = getattr(config, "conv_kernel_size", 31)
|
| 85 |
+
offsets = config.window_offsets
|
| 86 |
+
|
| 87 |
+
# Macaron FFN sandwich
|
| 88 |
+
self.ff1 = _macaron_ffn(d, d_ff, dropout)
|
| 89 |
+
self.ff2 = _macaron_ffn(d, d_ff, dropout)
|
| 90 |
+
|
| 91 |
+
# Grassmann mixer (replaces self-attention)
|
| 92 |
+
self.ln_mix = nn.LayerNorm(d)
|
| 93 |
+
self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps)
|
| 94 |
+
self.drop = nn.Dropout(dropout)
|
| 95 |
+
|
| 96 |
+
# ConvModule for local patterns
|
| 97 |
+
self.conv = ConvModule(d, kernel, dropout)
|
| 98 |
+
|
| 99 |
+
# Final LayerNorm
|
| 100 |
+
self.ln_out = nn.LayerNorm(d)
|
| 101 |
+
|
| 102 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 103 |
+
"""
|
| 104 |
+
Args:
|
| 105 |
+
x: (batch, seq_len, d_model)
|
| 106 |
+
Returns:
|
| 107 |
+
(batch, seq_len, d_model)
|
| 108 |
+
"""
|
| 109 |
+
# Half-step FFN 1
|
| 110 |
+
x = x + 0.5 * self.ff1(x)
|
| 111 |
+
# Grassmann mixer (with gate as residual)
|
| 112 |
+
x = x + self.drop(self.mixer(self.ln_mix(x)))
|
| 113 |
+
# ConvModule
|
| 114 |
+
x = x + self.conv(x)
|
| 115 |
+
# Half-step FFN 2 + final norm
|
| 116 |
+
return self.ln_out(x + 0.5 * self.ff2(x))
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class AblationConformerBlock(nn.Module):
|
| 120 |
+
"""
|
| 121 |
+
Conformer block with symmetric-product mixer (ablation baseline).
|
| 122 |
+
Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products.
|
| 123 |
+
"""
|
| 124 |
+
|
| 125 |
+
def __init__(self, config: GrassmannConfig):
|
| 126 |
+
super().__init__()
|
| 127 |
+
d = config.d_model
|
| 128 |
+
d_ff = config.d_ff
|
| 129 |
+
r = config.r_reduced
|
| 130 |
+
dropout = config.dropout
|
| 131 |
+
kernel = getattr(config, "conv_kernel_size", 31)
|
| 132 |
+
offsets = config.window_offsets
|
| 133 |
+
|
| 134 |
+
self.ff1 = _macaron_ffn(d, d_ff, dropout)
|
| 135 |
+
self.ff2 = _macaron_ffn(d, d_ff, dropout)
|
| 136 |
+
|
| 137 |
+
self.ln_mix = nn.LayerNorm(d)
|
| 138 |
+
self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps)
|
| 139 |
+
self.drop = nn.Dropout(dropout)
|
| 140 |
+
|
| 141 |
+
self.conv = ConvModule(d, kernel, dropout)
|
| 142 |
+
self.ln_out = nn.LayerNorm(d)
|
| 143 |
+
|
| 144 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 145 |
+
x = x + 0.5 * self.ff1(x)
|
| 146 |
+
x = x + self.drop(self.mixer(self.ln_mix(x)))
|
| 147 |
+
x = x + self.conv(x)
|
| 148 |
+
return self.ln_out(x + 0.5 * self.ff2(x))
|
src/models/conv_module.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Conformer ConvModule.
|
| 3 |
+
|
| 4 |
+
Captures local temporal patterns that Grassmann mixing misses.
|
| 5 |
+
From Gulati et al. 2020 (arXiv 2005.08100):
|
| 6 |
+
|
| 7 |
+
LayerNorm → Pointwise Conv (GLU) → Depthwise Conv → BatchNorm → SiLU → Pointwise Conv → Dropout
|
| 8 |
+
|
| 9 |
+
This is the key component that makes Conformer superior to pure
|
| 10 |
+
Transformer/Grassmann for speech — it captures local phonetic patterns
|
| 11 |
+
within a convolutional receptive field.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import torch
|
| 15 |
+
import torch.nn as nn
|
| 16 |
+
import torch.nn.functional as F
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class ConvModule(nn.Module):
|
| 20 |
+
"""Conformer convolution module with depthwise separable convolution."""
|
| 21 |
+
|
| 22 |
+
def __init__(self, d_model: int, kernel_size: int = 31, dropout: float = 0.1):
|
| 23 |
+
super().__init__()
|
| 24 |
+
self.ln = nn.LayerNorm(d_model)
|
| 25 |
+
# Pointwise expansion with GLU gating
|
| 26 |
+
self.pw1 = nn.Conv1d(d_model, 2 * d_model, kernel_size=1)
|
| 27 |
+
# Depthwise convolution (each channel independently)
|
| 28 |
+
self.dw = nn.Conv1d(
|
| 29 |
+
d_model, d_model, kernel_size,
|
| 30 |
+
padding=kernel_size // 2, groups=d_model,
|
| 31 |
+
)
|
| 32 |
+
self.bn = nn.BatchNorm1d(d_model)
|
| 33 |
+
# Pointwise projection back
|
| 34 |
+
self.pw2 = nn.Conv1d(d_model, d_model, kernel_size=1)
|
| 35 |
+
self.drop = nn.Dropout(dropout)
|
| 36 |
+
|
| 37 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 38 |
+
"""
|
| 39 |
+
Args:
|
| 40 |
+
x: (batch, seq_len, d_model)
|
| 41 |
+
Returns:
|
| 42 |
+
(batch, seq_len, d_model)
|
| 43 |
+
"""
|
| 44 |
+
# LayerNorm → transpose for Conv1d
|
| 45 |
+
y = self.ln(x).transpose(1, 2) # (B, d, T)
|
| 46 |
+
# Pointwise expansion + GLU gate
|
| 47 |
+
y = F.glu(self.pw1(y), dim=1) # (B, d, T)
|
| 48 |
+
# Depthwise conv + BatchNorm + SiLU
|
| 49 |
+
y = F.silu(self.bn(self.dw(y))) # (B, d, T)
|
| 50 |
+
# Pointwise projection + dropout
|
| 51 |
+
y = self.drop(self.pw2(y)) # (B, d, T)
|
| 52 |
+
return y.transpose(1, 2) # (B, T, d)
|
src/models/grassmann_cross_layer.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Grassmann Cross-Mixing Layer.
|
| 3 |
+
|
| 4 |
+
Replaces cross-attention in the decoder with Grassmann geometry.
|
| 5 |
+
Instead of Q*K^T dot products between decoder and encoder states,
|
| 6 |
+
we form Plücker coordinates between decoder positions and encoder
|
| 7 |
+
positions within a local monotonic window.
|
| 8 |
+
|
| 9 |
+
This is the key novel component — cross-sequence interaction via
|
| 10 |
+
subspace geometry on Gr(2, r) rather than attention weights.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn as nn
|
| 15 |
+
|
| 16 |
+
from .grassmann_layer import PluckerEncoder
|
| 17 |
+
from .config import GrassmannConfig
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class GrassmannCrossLayer(nn.Module):
|
| 21 |
+
"""
|
| 22 |
+
Cross-mixing between decoder and encoder via Grassmann/Plücker geometry.
|
| 23 |
+
|
| 24 |
+
For each decoder position t, estimates an alignment position in the
|
| 25 |
+
encoder sequence, then computes Plücker features between the decoder
|
| 26 |
+
state and encoder states in a window around that alignment.
|
| 27 |
+
|
| 28 |
+
This replaces cross-attention entirely — no Q*K^T, no softmax,
|
| 29 |
+
just subspace geometry on a manifold.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
def __init__(self, config: GrassmannConfig):
|
| 33 |
+
super().__init__()
|
| 34 |
+
d = config.d_model
|
| 35 |
+
r = config.r_reduced
|
| 36 |
+
plucker_dim = config.plucker_dim
|
| 37 |
+
|
| 38 |
+
# Separate reduction projections for decoder and encoder
|
| 39 |
+
self.W_red_dec = nn.Linear(d, r)
|
| 40 |
+
self.W_red_enc = nn.Linear(d, r)
|
| 41 |
+
|
| 42 |
+
# Plücker encoder (shared geometry)
|
| 43 |
+
self.plucker = PluckerEncoder(r, eps=config.plucker_eps)
|
| 44 |
+
|
| 45 |
+
# Project Plücker features back to model dim
|
| 46 |
+
self.W_plu = nn.Linear(plucker_dim, d)
|
| 47 |
+
|
| 48 |
+
# Gated fusion: blend decoder state with cross-mixed features
|
| 49 |
+
self.W_gate = nn.Linear(2 * d, d)
|
| 50 |
+
|
| 51 |
+
# Layer norm
|
| 52 |
+
self.norm = nn.LayerNorm(d)
|
| 53 |
+
self.dropout = nn.Dropout(config.dropout)
|
| 54 |
+
|
| 55 |
+
# Cross-mixing window: how many encoder positions to look at
|
| 56 |
+
# per decoder position (on each side of the alignment point)
|
| 57 |
+
self.cross_window = config.cross_window
|
| 58 |
+
|
| 59 |
+
def _estimate_alignment(
|
| 60 |
+
self, dec_len: int, enc_len: int, device: torch.device
|
| 61 |
+
) -> torch.Tensor:
|
| 62 |
+
"""
|
| 63 |
+
Estimate monotonic alignment: decoder position t maps to
|
| 64 |
+
encoder position t * (enc_len / dec_len).
|
| 65 |
+
|
| 66 |
+
Returns integer encoder indices for each decoder position.
|
| 67 |
+
"""
|
| 68 |
+
# Linear interpolation: dec pos 0..T_dec maps to enc pos 0..T_enc
|
| 69 |
+
dec_positions = torch.arange(dec_len, device=device, dtype=torch.float)
|
| 70 |
+
scale = enc_len / max(dec_len, 1)
|
| 71 |
+
enc_positions = (dec_positions * scale).long().clamp(0, enc_len - 1)
|
| 72 |
+
return enc_positions
|
| 73 |
+
|
| 74 |
+
def forward(
|
| 75 |
+
self, h_dec: torch.Tensor, h_enc: torch.Tensor
|
| 76 |
+
) -> torch.Tensor:
|
| 77 |
+
"""
|
| 78 |
+
Args:
|
| 79 |
+
h_dec: (batch, T_dec, d_model) - decoder hidden states
|
| 80 |
+
h_enc: (batch, T_enc, d_model) - encoder hidden states
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
h_out: (batch, T_dec, d_model) - cross-mixed decoder states
|
| 84 |
+
"""
|
| 85 |
+
B, T_dec, D = h_dec.shape
|
| 86 |
+
T_enc = h_enc.shape[1]
|
| 87 |
+
|
| 88 |
+
# Reduce both to Grassmann space
|
| 89 |
+
z_dec = self.W_red_dec(h_dec) # (B, T_dec, r)
|
| 90 |
+
z_enc = self.W_red_enc(h_enc) # (B, T_enc, r)
|
| 91 |
+
|
| 92 |
+
# Estimate alignment positions
|
| 93 |
+
align = self._estimate_alignment(T_dec, T_enc, h_dec.device) # (T_dec,)
|
| 94 |
+
|
| 95 |
+
# For each decoder position, gather encoder states in a window
|
| 96 |
+
# around the alignment point and compute Plücker features
|
| 97 |
+
w = self.cross_window
|
| 98 |
+
g_accum = torch.zeros_like(h_dec) # (B, T_dec, D)
|
| 99 |
+
count = 0
|
| 100 |
+
|
| 101 |
+
for offset in range(-w, w + 1):
|
| 102 |
+
# Encoder indices for this offset
|
| 103 |
+
enc_idx = (align + offset).clamp(0, T_enc - 1) # (T_dec,)
|
| 104 |
+
|
| 105 |
+
# Gather encoder states at these positions
|
| 106 |
+
enc_gathered = z_enc[:, enc_idx, :] # (B, T_dec, r)
|
| 107 |
+
|
| 108 |
+
# Plücker encoding between decoder and encoder states
|
| 109 |
+
p = self.plucker(z_dec, enc_gathered) # (B, T_dec, plucker_dim)
|
| 110 |
+
|
| 111 |
+
# Project back
|
| 112 |
+
g = self.W_plu(p) # (B, T_dec, D)
|
| 113 |
+
g_accum += g
|
| 114 |
+
count += 1
|
| 115 |
+
|
| 116 |
+
g_cross = g_accum / max(count, 1)
|
| 117 |
+
|
| 118 |
+
# Gated fusion
|
| 119 |
+
u = torch.cat([h_dec, g_cross], dim=-1) # (B, T_dec, 2D)
|
| 120 |
+
alpha = torch.sigmoid(self.W_gate(u)) # (B, T_dec, D)
|
| 121 |
+
h_mixed = alpha * h_dec + (1 - alpha) * g_cross
|
| 122 |
+
|
| 123 |
+
# Residual + layer norm + dropout
|
| 124 |
+
h_out = self.norm(h_dec + self.dropout(h_mixed))
|
| 125 |
+
|
| 126 |
+
return h_out
|
src/models/grassmann_decoder.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Grassmann Decoder.
|
| 3 |
+
|
| 4 |
+
A fully attention-free autoregressive decoder using:
|
| 5 |
+
1. Causal Grassmann self-mixing (same as encoder)
|
| 6 |
+
2. Grassmann cross-mixing (replaces cross-attention)
|
| 7 |
+
3. Position-wise FFN
|
| 8 |
+
|
| 9 |
+
Each decoder block: CausalSelfMix → CrossMix → FFN
|
| 10 |
+
Stacked M times, with token embeddings and output projection.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import math
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
|
| 18 |
+
from .config import GrassmannConfig
|
| 19 |
+
from .grassmann_layer import CausalGrassmannLayer
|
| 20 |
+
from .grassmann_cross_layer import GrassmannCrossLayer
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class GrassmannDecoderBlock(nn.Module):
|
| 24 |
+
"""
|
| 25 |
+
One decoder block: self-mixing → cross-mixing → FFN.
|
| 26 |
+
|
| 27 |
+
The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder).
|
| 28 |
+
The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder).
|
| 29 |
+
The FFN is already included in CausalGrassmannLayer, so we only add
|
| 30 |
+
the cross-mixing step between self-mixing and the FFN residual.
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
|
| 34 |
+
super().__init__()
|
| 35 |
+
d = config.d_model
|
| 36 |
+
|
| 37 |
+
# Causal self-mixing (includes FFN)
|
| 38 |
+
self.self_mix = CausalGrassmannLayer(config, layer_idx)
|
| 39 |
+
|
| 40 |
+
# Grassmann cross-mixing with encoder
|
| 41 |
+
self.cross_mix = GrassmannCrossLayer(config)
|
| 42 |
+
|
| 43 |
+
# Additional FFN after cross-mixing
|
| 44 |
+
self.ffn = nn.Sequential(
|
| 45 |
+
nn.Linear(d, config.d_ff),
|
| 46 |
+
nn.GELU(),
|
| 47 |
+
nn.Dropout(config.dropout),
|
| 48 |
+
nn.Linear(config.d_ff, d),
|
| 49 |
+
nn.Dropout(config.dropout),
|
| 50 |
+
)
|
| 51 |
+
self.norm_ffn = nn.LayerNorm(d)
|
| 52 |
+
|
| 53 |
+
def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor:
|
| 54 |
+
"""
|
| 55 |
+
Args:
|
| 56 |
+
h_dec: (batch, T_dec, d_model) - decoder states
|
| 57 |
+
h_enc: (batch, T_enc, d_model) - encoder output
|
| 58 |
+
|
| 59 |
+
Returns:
|
| 60 |
+
(batch, T_dec, d_model) - updated decoder states
|
| 61 |
+
"""
|
| 62 |
+
# Self-mixing (causal, within decoder sequence)
|
| 63 |
+
h = self.self_mix(h_dec)
|
| 64 |
+
|
| 65 |
+
# Cross-mixing (decoder ↔ encoder via Grassmann geometry)
|
| 66 |
+
h = self.cross_mix(h, h_enc)
|
| 67 |
+
|
| 68 |
+
# FFN with residual
|
| 69 |
+
h_out = self.norm_ffn(h + self.ffn(h))
|
| 70 |
+
|
| 71 |
+
return h_out
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class GrassmannDecoder(nn.Module):
|
| 75 |
+
"""
|
| 76 |
+
Full Grassmann decoder: token embeddings → M decoder blocks → output logits.
|
| 77 |
+
|
| 78 |
+
Fully attention-free. All sequence interactions happen through
|
| 79 |
+
Plücker coordinates on Gr(2, r).
|
| 80 |
+
"""
|
| 81 |
+
|
| 82 |
+
def __init__(self, config: GrassmannConfig):
|
| 83 |
+
super().__init__()
|
| 84 |
+
self.config = config
|
| 85 |
+
|
| 86 |
+
# Token embeddings
|
| 87 |
+
self.token_embed = nn.Embedding(config.vocab_size, config.d_model)
|
| 88 |
+
self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model)
|
| 89 |
+
self.embed_dropout = nn.Dropout(config.dropout)
|
| 90 |
+
self.embed_scale = math.sqrt(config.d_model)
|
| 91 |
+
|
| 92 |
+
# Decoder blocks
|
| 93 |
+
self.blocks = nn.ModuleList([
|
| 94 |
+
GrassmannDecoderBlock(config, i)
|
| 95 |
+
for i in range(config.n_decoder_layers)
|
| 96 |
+
])
|
| 97 |
+
|
| 98 |
+
# Output projection to vocab
|
| 99 |
+
self.output_norm = nn.LayerNorm(config.d_model)
|
| 100 |
+
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
|
| 101 |
+
|
| 102 |
+
def forward(
|
| 103 |
+
self,
|
| 104 |
+
tokens: torch.Tensor,
|
| 105 |
+
encoder_out: torch.Tensor,
|
| 106 |
+
) -> torch.Tensor:
|
| 107 |
+
"""
|
| 108 |
+
Args:
|
| 109 |
+
tokens: (batch, T_dec) - input token indices (shifted right)
|
| 110 |
+
encoder_out: (batch, T_enc, d_model) - encoder hidden states
|
| 111 |
+
|
| 112 |
+
Returns:
|
| 113 |
+
logits: (batch, T_dec, vocab_size)
|
| 114 |
+
"""
|
| 115 |
+
B, T = tokens.shape
|
| 116 |
+
|
| 117 |
+
# Token + positional embeddings
|
| 118 |
+
positions = torch.arange(T, device=tokens.device).unsqueeze(0)
|
| 119 |
+
h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions)
|
| 120 |
+
h = self.embed_dropout(h)
|
| 121 |
+
|
| 122 |
+
# Decoder blocks
|
| 123 |
+
for block in self.blocks:
|
| 124 |
+
h = block(h, encoder_out)
|
| 125 |
+
|
| 126 |
+
# Output
|
| 127 |
+
h = self.output_norm(h)
|
| 128 |
+
logits = self.output_proj(h)
|
| 129 |
+
|
| 130 |
+
return logits
|
src/models/grassmann_layer.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Causal Grassmann Mixing Layer.
|
| 3 |
+
|
| 4 |
+
Implements the core architecture from "Attention Is Not What You Need"
|
| 5 |
+
(arXiv:2512.19428). Replaces self-attention with:
|
| 6 |
+
|
| 7 |
+
1. Linear reduction: h_t ∈ R^d → z_t ∈ R^r
|
| 8 |
+
2. Multi-scale pairing: (z_t, z_{t+Δ}) for Δ ∈ W
|
| 9 |
+
3. Plücker encoding: p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i}
|
| 10 |
+
4. Projection back: g_t = W_plu @ p_hat + b_plu
|
| 11 |
+
5. Gated fusion: h_mix = α ⊙ h + (1-α) ⊙ g
|
| 12 |
+
6. Feed-forward block: standard FFN with GELU
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
import torch.nn.functional as F
|
| 18 |
+
|
| 19 |
+
from .config import GrassmannConfig
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class PluckerEncoder(nn.Module):
|
| 23 |
+
"""Computes Plücker coordinates for pairs of reduced vectors."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, r: int, eps: float = 1e-8):
|
| 26 |
+
super().__init__()
|
| 27 |
+
self.r = r
|
| 28 |
+
self.eps = eps
|
| 29 |
+
# Precompute index pairs (i, j) where i < j
|
| 30 |
+
idx_i, idx_j = [], []
|
| 31 |
+
for i in range(r):
|
| 32 |
+
for j in range(i + 1, r):
|
| 33 |
+
idx_i.append(i)
|
| 34 |
+
idx_j.append(j)
|
| 35 |
+
self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
|
| 36 |
+
self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
|
| 37 |
+
|
| 38 |
+
def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
|
| 39 |
+
"""
|
| 40 |
+
Compute normalized Plücker coordinates.
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
z_t: (batch, seq, r) - reduced states at position t
|
| 44 |
+
z_delta: (batch, seq, r) - reduced states at position t+Δ
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
(batch, seq, C(r,2)) - normalized Plücker vectors
|
| 48 |
+
"""
|
| 49 |
+
# p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i}
|
| 50 |
+
p = (
|
| 51 |
+
z_t[..., self.idx_i] * z_delta[..., self.idx_j]
|
| 52 |
+
- z_t[..., self.idx_j] * z_delta[..., self.idx_i]
|
| 53 |
+
)
|
| 54 |
+
# Normalize for numerical stability
|
| 55 |
+
norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
|
| 56 |
+
return p / norm
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class GrassmannMixer(nn.Module):
|
| 60 |
+
"""
|
| 61 |
+
Grassmann/Plücker mixer — the core sequence interaction.
|
| 62 |
+
|
| 63 |
+
Supports:
|
| 64 |
+
- Causal (one-sided) or bidirectional (two-sided) mixing
|
| 65 |
+
- Mean-first (anchored) form: 3.5x faster, mathematically exact
|
| 66 |
+
- Standard (as-written) form: project then average
|
| 67 |
+
|
| 68 |
+
Two-sided mixing is appropriate for CTC/encoder mode since speech
|
| 69 |
+
isn't generated left-to-right. Antisymmetry of Plücker coordinates
|
| 70 |
+
naturally distinguishes past from future even in two-sided mode.
|
| 71 |
+
"""
|
| 72 |
+
|
| 73 |
+
def __init__(self, d: int, r: int, offsets: list[int],
|
| 74 |
+
two_sided: bool = False, eps: float = 1e-8):
|
| 75 |
+
super().__init__()
|
| 76 |
+
self.offsets = offsets
|
| 77 |
+
self.two_sided = two_sided
|
| 78 |
+
self.eps = eps
|
| 79 |
+
plucker_dim = r * (r - 1) // 2
|
| 80 |
+
|
| 81 |
+
self.W_red = nn.Linear(d, r)
|
| 82 |
+
self.plucker = PluckerEncoder(r, eps=eps)
|
| 83 |
+
self.W_plu = nn.Linear(plucker_dim, d)
|
| 84 |
+
|
| 85 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 86 |
+
"""
|
| 87 |
+
Mean-first (anchored) form: average partners before projection.
|
| 88 |
+
Mathematically equivalent to as-written, but ~3.5x faster on GPU
|
| 89 |
+
because it does one W_plu matmul instead of one per offset.
|
| 90 |
+
"""
|
| 91 |
+
B, L, D = h.shape
|
| 92 |
+
z = self.W_red(h) # (B, L, r)
|
| 93 |
+
|
| 94 |
+
# Accumulate weighted partner vectors, then compute Plücker once
|
| 95 |
+
partner = torch.zeros_like(z) # (B, L, r)
|
| 96 |
+
count = z.new_zeros(1, L, 1)
|
| 97 |
+
|
| 98 |
+
for delta in self.offsets:
|
| 99 |
+
if delta >= L:
|
| 100 |
+
continue
|
| 101 |
+
earlier = z[:, :L - delta, :]
|
| 102 |
+
later = z[:, delta:, :]
|
| 103 |
+
|
| 104 |
+
# Compute norm for weighting (from dot products, numerically stable)
|
| 105 |
+
p = self.plucker(earlier, later)
|
| 106 |
+
w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal()
|
| 107 |
+
|
| 108 |
+
# Assign to later position (causal: t+Δ sees t)
|
| 109 |
+
partner[:, delta:] += w * earlier
|
| 110 |
+
count[:, delta:] += 1
|
| 111 |
+
|
| 112 |
+
if self.two_sided:
|
| 113 |
+
# Also let earlier position see later (bidirectional)
|
| 114 |
+
# Plücker antisymmetry (flip sign) distinguishes direction
|
| 115 |
+
partner[:, :L - delta] += -w * later
|
| 116 |
+
count[:, :L - delta] += 1
|
| 117 |
+
|
| 118 |
+
# Average partners, compute Plücker features, project
|
| 119 |
+
count = count.clamp(min=1)
|
| 120 |
+
avg_partner = partner / count
|
| 121 |
+
# Compute final Plücker between each position and its avg partner
|
| 122 |
+
p_final = self.plucker(z, avg_partner)
|
| 123 |
+
return self.W_plu(p_final)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
class CausalGrassmannLayer(nn.Module):
|
| 127 |
+
"""
|
| 128 |
+
One layer of the Causal Grassmann Transformer.
|
| 129 |
+
|
| 130 |
+
Replaces the self-attention block with Grassmann/Plücker mixing,
|
| 131 |
+
followed by the standard position-wise FFN.
|
| 132 |
+
|
| 133 |
+
Supports two_sided mode for encoder/CTC use cases.
|
| 134 |
+
"""
|
| 135 |
+
|
| 136 |
+
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
|
| 137 |
+
super().__init__()
|
| 138 |
+
d = config.d_model
|
| 139 |
+
r = config.r_reduced
|
| 140 |
+
two_sided = getattr(config, "two_sided", False)
|
| 141 |
+
|
| 142 |
+
# Grassmann mixer
|
| 143 |
+
self.mixer = GrassmannMixer(
|
| 144 |
+
d, r, config.window_offsets,
|
| 145 |
+
two_sided=two_sided, eps=config.plucker_eps,
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
# Gated fusion
|
| 149 |
+
self.W_gate = nn.Linear(2 * d, d)
|
| 150 |
+
|
| 151 |
+
# Layer norm after mixing
|
| 152 |
+
self.norm_mix = nn.LayerNorm(d)
|
| 153 |
+
self.dropout_mix = nn.Dropout(config.dropout)
|
| 154 |
+
|
| 155 |
+
# Feed-forward block
|
| 156 |
+
self.ffn = nn.Sequential(
|
| 157 |
+
nn.Linear(d, config.d_ff),
|
| 158 |
+
nn.GELU(),
|
| 159 |
+
nn.Dropout(config.dropout),
|
| 160 |
+
nn.Linear(config.d_ff, d),
|
| 161 |
+
nn.Dropout(config.dropout),
|
| 162 |
+
)
|
| 163 |
+
self.norm_ffn = nn.LayerNorm(d)
|
| 164 |
+
|
| 165 |
+
def forward(self, h: torch.Tensor) -> torch.Tensor:
|
| 166 |
+
"""
|
| 167 |
+
Args:
|
| 168 |
+
h: (batch, seq_len, d_model)
|
| 169 |
+
Returns:
|
| 170 |
+
h': (batch, seq_len, d_model)
|
| 171 |
+
"""
|
| 172 |
+
# Grassmann mixing
|
| 173 |
+
g = self.mixer(h) # (B, L, D)
|
| 174 |
+
|
| 175 |
+
# Gated fusion: (1-α)(g-h) as residual (Dhruv's variant)
|
| 176 |
+
u = torch.cat([h, g], dim=-1)
|
| 177 |
+
alpha = torch.sigmoid(self.W_gate(u))
|
| 178 |
+
h_mix = h + (1 - alpha) * (g - h)
|
| 179 |
+
|
| 180 |
+
# Layer norm + dropout + residual
|
| 181 |
+
h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
|
| 182 |
+
|
| 183 |
+
# Feed-forward with residual
|
| 184 |
+
h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
|
| 185 |
+
|
| 186 |
+
return h_out
|
src/models/grassmann_model.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Full Grassmann Transcription Model.
|
| 3 |
+
|
| 4 |
+
Audio → Mel Spectrogram features → Grassmann Layers → CTC Output
|
| 5 |
+
|
| 6 |
+
Architecture:
|
| 7 |
+
1. Conv frontend: projects mel features into d_model
|
| 8 |
+
2. Positional encoding
|
| 9 |
+
3. N x CausalGrassmannLayer
|
| 10 |
+
4. Linear CTC head → vocab_size
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import math
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
import torch.nn.functional as F
|
| 18 |
+
|
| 19 |
+
from .config import GrassmannConfig
|
| 20 |
+
from .grassmann_layer import CausalGrassmannLayer
|
| 21 |
+
from .ablation_layer import AblationLayer
|
| 22 |
+
from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class ConvFrontend(nn.Module):
|
| 26 |
+
"""Conv1d feature extractor for mel spectrograms (legacy)."""
|
| 27 |
+
|
| 28 |
+
def __init__(self, n_mels: int, d_model: int):
|
| 29 |
+
super().__init__()
|
| 30 |
+
self.conv = nn.Sequential(
|
| 31 |
+
nn.Conv1d(n_mels, d_model // 2, kernel_size=3, stride=2, padding=1),
|
| 32 |
+
nn.GELU(),
|
| 33 |
+
nn.Conv1d(d_model // 2, d_model, kernel_size=3, stride=2, padding=1),
|
| 34 |
+
nn.GELU(),
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 38 |
+
out = self.conv(x) # (B, d_model, T')
|
| 39 |
+
return out.transpose(1, 2) # (B, T', d_model)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class Conv2dSubsampling(nn.Module):
|
| 43 |
+
"""
|
| 44 |
+
Conv2d subsampling from Conformer (Gulati et al. 2020).
|
| 45 |
+
|
| 46 |
+
Two 3x3 stride-2 convolutions: 10ms fbank frames → 40ms encoder frames.
|
| 47 |
+
Standard ASR frontend used by ESPnet, WeNet, etc.
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
def __init__(self, n_mels: int, channels: int, d_model: int):
|
| 51 |
+
super().__init__()
|
| 52 |
+
self.conv1 = nn.Conv2d(1, channels, kernel_size=3, stride=2)
|
| 53 |
+
self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, stride=2)
|
| 54 |
+
# After two 3x3 stride-2 convs: freq dim = ((n_mels - 1) // 2 - 1) // 2
|
| 55 |
+
freq_out = ((n_mels - 1) // 2 - 1) // 2
|
| 56 |
+
self.out = nn.Linear(channels * freq_out, d_model)
|
| 57 |
+
|
| 58 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 59 |
+
"""
|
| 60 |
+
Args:
|
| 61 |
+
x: (batch, n_mels, time) - mel spectrogram
|
| 62 |
+
|
| 63 |
+
Returns:
|
| 64 |
+
(batch, time', d_model)
|
| 65 |
+
"""
|
| 66 |
+
# Reshape: (B, n_mels, T) → (B, 1, T, n_mels) for Conv2d
|
| 67 |
+
x = x.transpose(1, 2).unsqueeze(1) # (B, 1, T, n_mels)
|
| 68 |
+
x = F.relu(self.conv1(x))
|
| 69 |
+
x = F.relu(self.conv2(x))
|
| 70 |
+
b, c, t, f = x.shape
|
| 71 |
+
x = x.transpose(1, 2).reshape(b, t, c * f) # (B, T', C*F)
|
| 72 |
+
return self.out(x) # (B, T', d_model)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class PositionalEncoding(nn.Module):
|
| 76 |
+
"""Sinusoidal positional encoding."""
|
| 77 |
+
|
| 78 |
+
def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
|
| 79 |
+
super().__init__()
|
| 80 |
+
self.dropout = nn.Dropout(dropout)
|
| 81 |
+
|
| 82 |
+
pe = torch.zeros(max_len, d_model)
|
| 83 |
+
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
|
| 84 |
+
div_term = torch.exp(
|
| 85 |
+
torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
|
| 86 |
+
)
|
| 87 |
+
pe[:, 0::2] = torch.sin(position * div_term)
|
| 88 |
+
pe[:, 1::2] = torch.cos(position * div_term)
|
| 89 |
+
self.register_buffer("pe", pe.unsqueeze(0)) # (1, max_len, d_model)
|
| 90 |
+
|
| 91 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 92 |
+
x = x + self.pe[:, : x.size(1)]
|
| 93 |
+
return self.dropout(x)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
class GrassmannTranscriptionModel(nn.Module):
|
| 97 |
+
"""
|
| 98 |
+
Full model: Audio features → Grassmann layers → CTC output.
|
| 99 |
+
|
| 100 |
+
Supports two encoder modes:
|
| 101 |
+
- Plain: CausalGrassmannLayer stack (original)
|
| 102 |
+
- Conformer: GrassmannConformerBlock stack (with ConvModule + Macaron FFN)
|
| 103 |
+
|
| 104 |
+
And two frontend modes:
|
| 105 |
+
- Conv1d (legacy)
|
| 106 |
+
- Conv2d subsampling (proper ASR, from Conformer)
|
| 107 |
+
"""
|
| 108 |
+
|
| 109 |
+
def __init__(self, config: GrassmannConfig):
|
| 110 |
+
super().__init__()
|
| 111 |
+
self.config = config
|
| 112 |
+
|
| 113 |
+
# Frontend: mel → d_model
|
| 114 |
+
if config.use_conformer:
|
| 115 |
+
self.frontend = Conv2dSubsampling(
|
| 116 |
+
config.n_mels, config.conv_channels, config.d_model
|
| 117 |
+
)
|
| 118 |
+
else:
|
| 119 |
+
self.frontend = ConvFrontend(config.n_mels, config.d_model)
|
| 120 |
+
|
| 121 |
+
# Positional encoding
|
| 122 |
+
self.pos_enc = PositionalEncoding(
|
| 123 |
+
config.d_model, config.max_seq_len, config.dropout
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
# Encoder layers
|
| 127 |
+
ablation = getattr(config, "use_ablation", False)
|
| 128 |
+
if config.use_conformer:
|
| 129 |
+
block_cls = AblationConformerBlock if ablation else GrassmannConformerBlock
|
| 130 |
+
self.layers = nn.ModuleList(
|
| 131 |
+
[block_cls(config) for _ in range(config.n_layers)]
|
| 132 |
+
)
|
| 133 |
+
else:
|
| 134 |
+
layer_cls = AblationLayer if ablation else CausalGrassmannLayer
|
| 135 |
+
self.layers = nn.ModuleList(
|
| 136 |
+
[layer_cls(config, i) for i in range(config.n_layers)]
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
# CTC output head
|
| 140 |
+
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
|
| 141 |
+
|
| 142 |
+
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
| 143 |
+
"""
|
| 144 |
+
Args:
|
| 145 |
+
mel: (batch, n_mels, time) - mel spectrogram
|
| 146 |
+
|
| 147 |
+
Returns:
|
| 148 |
+
log_probs: (time', batch, vocab_size) - CTC-formatted log probs
|
| 149 |
+
"""
|
| 150 |
+
# Conv frontend
|
| 151 |
+
x = self.frontend(mel) # (B, T', d_model)
|
| 152 |
+
|
| 153 |
+
# Positional encoding
|
| 154 |
+
x = self.pos_enc(x)
|
| 155 |
+
|
| 156 |
+
# Grassmann layers
|
| 157 |
+
for layer in self.layers:
|
| 158 |
+
x = layer(x)
|
| 159 |
+
|
| 160 |
+
# CTC head
|
| 161 |
+
logits = self.output_proj(x) # (B, T', vocab_size)
|
| 162 |
+
log_probs = logits.log_softmax(dim=-1)
|
| 163 |
+
|
| 164 |
+
# CTC expects (T, B, C)
|
| 165 |
+
return log_probs.transpose(0, 1)
|
| 166 |
+
|
| 167 |
+
def get_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
|
| 168 |
+
"""Calculate output sequence lengths after conv downsampling."""
|
| 169 |
+
lengths = input_lengths
|
| 170 |
+
if self.config.use_conformer:
|
| 171 |
+
# Conv2d subsampling: two 3x3 stride-2 convs (no padding)
|
| 172 |
+
# Each conv: out = (in - 3) // 2 + 1 = (in - 1) // 2
|
| 173 |
+
for _ in range(2):
|
| 174 |
+
lengths = (lengths - 1) // 2
|
| 175 |
+
else:
|
| 176 |
+
# Conv1d frontend: two stride-2 convs with padding=1
|
| 177 |
+
for _ in range(2):
|
| 178 |
+
lengths = (lengths + 1) // 2
|
| 179 |
+
return lengths
|
src/models/grassmann_seq2seq.py
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Grassmann Seq2Seq Model — Fully Attention-Free Encoder-Decoder.
|
| 3 |
+
|
| 4 |
+
Audio → Grassmann Encoder → Grassmann Decoder → Text
|
| 5 |
+
|
| 6 |
+
Every sequence interaction in this model happens through Plücker
|
| 7 |
+
coordinates on the Grassmann manifold Gr(2, r). There are zero
|
| 8 |
+
attention matrices anywhere in the architecture:
|
| 9 |
+
|
| 10 |
+
Encoder self-mixing: Plücker(z_t, z_{t+Δ}) — local causal windows
|
| 11 |
+
Decoder self-mixing: Plücker(z_t, z_{t+Δ}) — causal autoregressive
|
| 12 |
+
Decoder cross-mixing: Plücker(z_dec, z_enc) — monotonic alignment windows
|
| 13 |
+
|
| 14 |
+
Trained with teacher forcing and cross-entropy loss.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torch.nn as nn
|
| 19 |
+
|
| 20 |
+
from .config import GrassmannConfig
|
| 21 |
+
from .grassmann_model import ConvFrontend, PositionalEncoding
|
| 22 |
+
from .grassmann_layer import CausalGrassmannLayer
|
| 23 |
+
from .grassmann_decoder import GrassmannDecoder
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class GrassmannEncoder(nn.Module):
|
| 27 |
+
"""Audio encoder: mel spectrogram → Grassmann layers → encoder states."""
|
| 28 |
+
|
| 29 |
+
def __init__(self, config: GrassmannConfig):
|
| 30 |
+
super().__init__()
|
| 31 |
+
self.frontend = ConvFrontend(config.n_mels, config.d_model)
|
| 32 |
+
self.pos_enc = PositionalEncoding(
|
| 33 |
+
config.d_model, config.max_seq_len, config.dropout
|
| 34 |
+
)
|
| 35 |
+
self.layers = nn.ModuleList([
|
| 36 |
+
CausalGrassmannLayer(config, i) for i in range(config.n_layers)
|
| 37 |
+
])
|
| 38 |
+
|
| 39 |
+
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
| 40 |
+
"""
|
| 41 |
+
Args:
|
| 42 |
+
mel: (batch, n_mels, time) - mel spectrogram
|
| 43 |
+
|
| 44 |
+
Returns:
|
| 45 |
+
(batch, T_enc, d_model) - encoder hidden states
|
| 46 |
+
"""
|
| 47 |
+
x = self.frontend(mel)
|
| 48 |
+
x = self.pos_enc(x)
|
| 49 |
+
for layer in self.layers:
|
| 50 |
+
x = layer(x)
|
| 51 |
+
return x
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class GrassmannSeq2SeqModel(nn.Module):
|
| 55 |
+
"""
|
| 56 |
+
Fully attention-free encoder-decoder for audio transcription.
|
| 57 |
+
|
| 58 |
+
Architecture:
|
| 59 |
+
Encoder: ConvFrontend → PosEnc → N × CausalGrassmannLayer
|
| 60 |
+
Decoder: TokenEmbed → PosEmbed → M × (CausalGrassmann + CrossGrassmann + FFN)
|
| 61 |
+
Output: Linear → vocab logits
|
| 62 |
+
|
| 63 |
+
All interactions use Plücker coordinates. Zero attention matrices.
|
| 64 |
+
"""
|
| 65 |
+
|
| 66 |
+
# Special token IDs
|
| 67 |
+
SOS_ID = 0 # Start of sequence (overloads blank/pad)
|
| 68 |
+
EOS_ID = 0 # End of sequence (same as SOS for simplicity)
|
| 69 |
+
PAD_ID = 0 # Padding
|
| 70 |
+
|
| 71 |
+
def __init__(self, config: GrassmannConfig):
|
| 72 |
+
super().__init__()
|
| 73 |
+
self.config = config
|
| 74 |
+
self.encoder = GrassmannEncoder(config)
|
| 75 |
+
self.decoder = GrassmannDecoder(config)
|
| 76 |
+
|
| 77 |
+
def forward(
|
| 78 |
+
self,
|
| 79 |
+
mel: torch.Tensor,
|
| 80 |
+
target_tokens: torch.Tensor,
|
| 81 |
+
) -> torch.Tensor:
|
| 82 |
+
"""
|
| 83 |
+
Teacher-forced forward pass.
|
| 84 |
+
|
| 85 |
+
Args:
|
| 86 |
+
mel: (batch, n_mels, time) - input mel spectrogram
|
| 87 |
+
target_tokens: (batch, T_target) - target token indices
|
| 88 |
+
|
| 89 |
+
Returns:
|
| 90 |
+
logits: (batch, T_target, vocab_size)
|
| 91 |
+
"""
|
| 92 |
+
# Encode audio
|
| 93 |
+
encoder_out = self.encoder(mel) # (B, T_enc, d)
|
| 94 |
+
|
| 95 |
+
# Decode with teacher forcing
|
| 96 |
+
# Input to decoder is target shifted right (prepend SOS)
|
| 97 |
+
dec_input = self._shift_right(target_tokens)
|
| 98 |
+
logits = self.decoder(dec_input, encoder_out)
|
| 99 |
+
|
| 100 |
+
return logits
|
| 101 |
+
|
| 102 |
+
def _shift_right(self, tokens: torch.Tensor) -> torch.Tensor:
|
| 103 |
+
"""Shift tokens right and prepend SOS token for teacher forcing."""
|
| 104 |
+
sos = torch.full(
|
| 105 |
+
(tokens.shape[0], 1), self.SOS_ID,
|
| 106 |
+
dtype=tokens.dtype, device=tokens.device
|
| 107 |
+
)
|
| 108 |
+
# Prepend SOS, drop last token
|
| 109 |
+
return torch.cat([sos, tokens[:, :-1]], dim=1)
|
| 110 |
+
|
| 111 |
+
def get_encoder_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
|
| 112 |
+
"""Calculate encoder output lengths after conv downsampling."""
|
| 113 |
+
lengths = input_lengths
|
| 114 |
+
for _ in range(2): # 2 conv layers with stride 2
|
| 115 |
+
lengths = (lengths + 1) // 2
|
| 116 |
+
return lengths
|
| 117 |
+
|
| 118 |
+
@torch.no_grad()
|
| 119 |
+
def greedy_decode(
|
| 120 |
+
self,
|
| 121 |
+
mel: torch.Tensor,
|
| 122 |
+
max_len: int = 200,
|
| 123 |
+
eos_id: int = 0,
|
| 124 |
+
repetition_penalty: float = 1.5,
|
| 125 |
+
no_repeat_ngram: int = 3,
|
| 126 |
+
) -> torch.Tensor:
|
| 127 |
+
"""
|
| 128 |
+
Autoregressive greedy decoding with repetition suppression.
|
| 129 |
+
|
| 130 |
+
Args:
|
| 131 |
+
mel: (1, n_mels, time) - single mel spectrogram
|
| 132 |
+
max_len: maximum output length
|
| 133 |
+
eos_id: end-of-sequence token ID
|
| 134 |
+
repetition_penalty: penalize tokens that already appeared (>1.0 to suppress)
|
| 135 |
+
no_repeat_ngram: block any n-gram that already occurred
|
| 136 |
+
|
| 137 |
+
Returns:
|
| 138 |
+
(seq_len,) - predicted token indices
|
| 139 |
+
"""
|
| 140 |
+
self.eval()
|
| 141 |
+
device = mel.device
|
| 142 |
+
|
| 143 |
+
# Encode once
|
| 144 |
+
encoder_out = self.encoder(mel) # (1, T_enc, d)
|
| 145 |
+
|
| 146 |
+
# Start with SOS token
|
| 147 |
+
generated = [self.SOS_ID]
|
| 148 |
+
|
| 149 |
+
for _ in range(max_len):
|
| 150 |
+
# Build decoder input from generated tokens so far
|
| 151 |
+
dec_input = torch.tensor(
|
| 152 |
+
[generated], dtype=torch.long, device=device
|
| 153 |
+
) # (1, t)
|
| 154 |
+
|
| 155 |
+
# Decode
|
| 156 |
+
logits = self.decoder(dec_input, encoder_out) # (1, t, vocab)
|
| 157 |
+
next_logits = logits[0, -1, :] # (vocab,)
|
| 158 |
+
|
| 159 |
+
# Repetition penalty: reduce logits for tokens already generated
|
| 160 |
+
if repetition_penalty != 1.0:
|
| 161 |
+
for prev_token in set(generated):
|
| 162 |
+
if next_logits[prev_token] > 0:
|
| 163 |
+
next_logits[prev_token] /= repetition_penalty
|
| 164 |
+
else:
|
| 165 |
+
next_logits[prev_token] *= repetition_penalty
|
| 166 |
+
|
| 167 |
+
# No-repeat n-gram blocking
|
| 168 |
+
if no_repeat_ngram > 0 and len(generated) >= no_repeat_ngram:
|
| 169 |
+
# Check if any token would create a repeated n-gram
|
| 170 |
+
ngram_prefix = generated[-(no_repeat_ngram - 1):]
|
| 171 |
+
for i in range(len(generated) - no_repeat_ngram + 1):
|
| 172 |
+
if generated[i:i + no_repeat_ngram - 1] == ngram_prefix:
|
| 173 |
+
blocked_token = generated[i + no_repeat_ngram - 1]
|
| 174 |
+
next_logits[blocked_token] = float("-inf")
|
| 175 |
+
|
| 176 |
+
next_token = next_logits.argmax().item()
|
| 177 |
+
generated.append(next_token)
|
| 178 |
+
|
| 179 |
+
if next_token == eos_id:
|
| 180 |
+
break
|
| 181 |
+
|
| 182 |
+
# Return without the initial SOS
|
| 183 |
+
return torch.tensor(generated[1:], dtype=torch.long)
|
src/repositories/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .dataset_repository import DatasetRepository
|
| 2 |
+
from .cache_repository import CacheRepository
|
src/repositories/cache_repository.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Cache Repository — disk and HF Hub persistence for pre-computed mel spectrograms.
|
| 3 |
+
|
| 4 |
+
Handles reading/writing individual .pt files on local disk, packing/unpacking
|
| 5 |
+
shards for HF Hub upload/download. Supports incremental shard uploads so
|
| 6 |
+
progress survives container restarts on ephemeral environments (HF Spaces).
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import gc
|
| 12 |
+
import json
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Iterator
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class CacheRepository:
|
| 20 |
+
"""Manages a local disk cache of (mel, tokens) .pt files and
|
| 21 |
+
syncs with a HuggingFace Dataset repo for persistence."""
|
| 22 |
+
|
| 23 |
+
def __init__(self, cache_dir: str):
|
| 24 |
+
self._cache_dir = Path(cache_dir)
|
| 25 |
+
self._shard_dir = self._cache_dir / "_shards"
|
| 26 |
+
|
| 27 |
+
@property
|
| 28 |
+
def path(self) -> str:
|
| 29 |
+
return str(self._cache_dir)
|
| 30 |
+
|
| 31 |
+
def exists(self) -> bool:
|
| 32 |
+
"""Check if cache has any .pt files."""
|
| 33 |
+
return self._cache_dir.exists() and any(self._cache_dir.glob("*.pt"))
|
| 34 |
+
|
| 35 |
+
def count(self) -> int:
|
| 36 |
+
"""Number of cached samples."""
|
| 37 |
+
if not self._cache_dir.exists():
|
| 38 |
+
return 0
|
| 39 |
+
return len(list(self._cache_dir.glob("*.pt")))
|
| 40 |
+
|
| 41 |
+
def indices(self) -> list[int]:
|
| 42 |
+
"""Sorted list of all cached sample indices."""
|
| 43 |
+
if not self._cache_dir.exists():
|
| 44 |
+
return []
|
| 45 |
+
return sorted(int(f.stem) for f in self._cache_dir.glob("*.pt"))
|
| 46 |
+
|
| 47 |
+
def save(self, idx: int, mel: torch.Tensor, tokens: torch.Tensor) -> None:
|
| 48 |
+
"""Save a single (mel, tokens) pair to disk."""
|
| 49 |
+
self._cache_dir.mkdir(parents=True, exist_ok=True)
|
| 50 |
+
torch.save((mel, tokens), self._cache_dir / f"{idx}.pt")
|
| 51 |
+
|
| 52 |
+
def load(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 53 |
+
"""Load a single (mel, tokens) pair from disk."""
|
| 54 |
+
mel, tokens = torch.load(
|
| 55 |
+
self._cache_dir / f"{idx}.pt",
|
| 56 |
+
weights_only=True, map_location="cpu",
|
| 57 |
+
)
|
| 58 |
+
mel_out = mel.contiguous().clone()
|
| 59 |
+
tok_out = tokens.contiguous().clone()
|
| 60 |
+
del mel, tokens
|
| 61 |
+
return mel_out, tok_out
|
| 62 |
+
|
| 63 |
+
def save_stream(self, stream: Iterator[tuple[torch.Tensor, torch.Tensor]],
|
| 64 |
+
source_label: str = "",
|
| 65 |
+
repo_id: str | None = None,
|
| 66 |
+
samples_per_shard: int = 50000) -> int:
|
| 67 |
+
"""Save an iterator of (mel, tokens) pairs to disk sequentially.
|
| 68 |
+
|
| 69 |
+
When repo_id is set, uploads a shard to HF Hub every samples_per_shard
|
| 70 |
+
samples and deletes the local .pt files. This makes caching resilient
|
| 71 |
+
to container restarts — progress is persisted incrementally.
|
| 72 |
+
|
| 73 |
+
Returns the number of samples saved.
|
| 74 |
+
"""
|
| 75 |
+
self._cache_dir.mkdir(parents=True, exist_ok=True)
|
| 76 |
+
existing = self.indices()
|
| 77 |
+
start_idx = (max(existing) + 1) if existing else 0
|
| 78 |
+
count = 0
|
| 79 |
+
shard_buffer: list[tuple[torch.Tensor, torch.Tensor]] = []
|
| 80 |
+
|
| 81 |
+
# Figure out next shard index from local + remote shards
|
| 82 |
+
shard_idx = self._next_shard_idx(repo_id=repo_id)
|
| 83 |
+
|
| 84 |
+
for mel, tokens in stream:
|
| 85 |
+
self.save(start_idx + count, mel, tokens)
|
| 86 |
+
shard_buffer.append((mel.clone(), tokens.clone()))
|
| 87 |
+
count += 1
|
| 88 |
+
del mel, tokens
|
| 89 |
+
|
| 90 |
+
if count % 1000 == 0:
|
| 91 |
+
label = f"{source_label}: " if source_label else ""
|
| 92 |
+
print(f" {label}{count} samples cached ({start_idx + count} total)...",
|
| 93 |
+
flush=True)
|
| 94 |
+
gc.collect()
|
| 95 |
+
|
| 96 |
+
# Incremental upload when shard is full
|
| 97 |
+
if repo_id and len(shard_buffer) >= samples_per_shard:
|
| 98 |
+
self._upload_shard(repo_id, shard_idx, shard_buffer)
|
| 99 |
+
# Delete local .pt files that were just uploaded
|
| 100 |
+
self._delete_range(start_idx + count - len(shard_buffer),
|
| 101 |
+
start_idx + count)
|
| 102 |
+
shard_buffer = []
|
| 103 |
+
shard_idx += 1
|
| 104 |
+
gc.collect()
|
| 105 |
+
|
| 106 |
+
# Upload remaining partial shard
|
| 107 |
+
if repo_id and shard_buffer:
|
| 108 |
+
self._upload_shard(repo_id, shard_idx, shard_buffer)
|
| 109 |
+
self._delete_range(start_idx + count - len(shard_buffer),
|
| 110 |
+
start_idx + count)
|
| 111 |
+
shard_buffer = []
|
| 112 |
+
shard_idx += 1
|
| 113 |
+
|
| 114 |
+
gc.collect()
|
| 115 |
+
return count
|
| 116 |
+
|
| 117 |
+
def _next_shard_idx(self, repo_id: str | None = None) -> int:
|
| 118 |
+
"""Find the next shard index from local AND remote shard files.
|
| 119 |
+
|
| 120 |
+
Checks both local _shards/ directory and the HF Hub repo to avoid
|
| 121 |
+
overwriting shards that were uploaded before a container restart.
|
| 122 |
+
"""
|
| 123 |
+
indices: list[int] = []
|
| 124 |
+
|
| 125 |
+
# Local shards
|
| 126 |
+
self._shard_dir.mkdir(parents=True, exist_ok=True)
|
| 127 |
+
for f in self._shard_dir.glob("shard_*.pt"):
|
| 128 |
+
try:
|
| 129 |
+
indices.append(int(f.stem.split("_")[1]))
|
| 130 |
+
except (IndexError, ValueError):
|
| 131 |
+
pass
|
| 132 |
+
|
| 133 |
+
# Remote shards (survives container restarts)
|
| 134 |
+
if repo_id:
|
| 135 |
+
try:
|
| 136 |
+
from huggingface_hub import list_repo_files
|
| 137 |
+
files = list_repo_files(repo_id, repo_type="dataset")
|
| 138 |
+
for fname in files:
|
| 139 |
+
if fname.startswith("shard_") and fname.endswith(".pt"):
|
| 140 |
+
try:
|
| 141 |
+
indices.append(int(fname.split("_")[1].split(".")[0]))
|
| 142 |
+
except (IndexError, ValueError):
|
| 143 |
+
pass
|
| 144 |
+
except Exception:
|
| 145 |
+
pass
|
| 146 |
+
|
| 147 |
+
return (max(indices) + 1) if indices else 0
|
| 148 |
+
|
| 149 |
+
def _upload_shard(self, repo_id: str, shard_idx: int,
|
| 150 |
+
data: list[tuple[torch.Tensor, torch.Tensor]]) -> None:
|
| 151 |
+
"""Pack and upload a single shard to HF Hub."""
|
| 152 |
+
from huggingface_hub import HfApi, create_repo
|
| 153 |
+
|
| 154 |
+
api = HfApi()
|
| 155 |
+
try:
|
| 156 |
+
create_repo(repo_id, repo_type="dataset", exist_ok=True)
|
| 157 |
+
except Exception:
|
| 158 |
+
pass
|
| 159 |
+
|
| 160 |
+
self._shard_dir.mkdir(parents=True, exist_ok=True)
|
| 161 |
+
shard_path = self._shard_dir / f"shard_{shard_idx:04d}.pt"
|
| 162 |
+
torch.save(data, shard_path)
|
| 163 |
+
size_mb = shard_path.stat().st_size / 1e6
|
| 164 |
+
print(f" Uploading shard_{shard_idx:04d}: {len(data)} samples ({size_mb:.0f} MB)...",
|
| 165 |
+
flush=True)
|
| 166 |
+
|
| 167 |
+
api.upload_file(
|
| 168 |
+
path_or_fileobj=str(shard_path),
|
| 169 |
+
path_in_repo=f"shard_{shard_idx:04d}.pt",
|
| 170 |
+
repo_id=repo_id,
|
| 171 |
+
repo_type="dataset",
|
| 172 |
+
commit_message=f"Shard {shard_idx}: {len(data)} samples",
|
| 173 |
+
)
|
| 174 |
+
# Delete local shard after upload to save disk
|
| 175 |
+
shard_path.unlink(missing_ok=True)
|
| 176 |
+
print(f" Shard {shard_idx} uploaded and cleaned up.", flush=True)
|
| 177 |
+
|
| 178 |
+
def _delete_range(self, start: int, end: int) -> None:
|
| 179 |
+
"""Delete .pt files in [start, end) to free disk after shard upload."""
|
| 180 |
+
for idx in range(start, end):
|
| 181 |
+
pt = self._cache_dir / f"{idx}.pt"
|
| 182 |
+
pt.unlink(missing_ok=True)
|
| 183 |
+
|
| 184 |
+
def download_from_hub(self, repo_id: str) -> int:
|
| 185 |
+
"""Download pre-cached mel shards from a HF Dataset repo and unpack to .pt files.
|
| 186 |
+
|
| 187 |
+
Returns the total number of unpacked samples.
|
| 188 |
+
"""
|
| 189 |
+
from huggingface_hub import hf_hub_download, list_repo_files
|
| 190 |
+
|
| 191 |
+
if self.exists():
|
| 192 |
+
n = self.count()
|
| 193 |
+
print(f" Found {n} cached samples in {self._cache_dir}, skipping download.")
|
| 194 |
+
return n
|
| 195 |
+
|
| 196 |
+
self._cache_dir.mkdir(parents=True, exist_ok=True)
|
| 197 |
+
print(f" Downloading mel cache from {repo_id}...")
|
| 198 |
+
|
| 199 |
+
files = list_repo_files(repo_id, repo_type="dataset")
|
| 200 |
+
shard_files = sorted(f for f in files if f.startswith("shard_") and f.endswith(".pt"))
|
| 201 |
+
print(f" Found {len(shard_files)} shards")
|
| 202 |
+
|
| 203 |
+
sample_idx = 0
|
| 204 |
+
for shard_name in shard_files:
|
| 205 |
+
local_path = hf_hub_download(repo_id, shard_name, repo_type="dataset")
|
| 206 |
+
shard_data = torch.load(local_path, weights_only=True, map_location="cpu")
|
| 207 |
+
for mel, tokens in shard_data:
|
| 208 |
+
torch.save((mel, tokens), self._cache_dir / f"{sample_idx}.pt")
|
| 209 |
+
sample_idx += 1
|
| 210 |
+
del shard_data
|
| 211 |
+
print(f" Unpacked {shard_name} -> {sample_idx} samples total")
|
| 212 |
+
|
| 213 |
+
print(f" Downloaded and unpacked {sample_idx} samples to {self._cache_dir}")
|
| 214 |
+
return sample_idx
|
| 215 |
+
|
| 216 |
+
def upload_to_hub(self, repo_id: str, samples_per_shard: int = 50000) -> None:
|
| 217 |
+
"""Pack cached .pt files into shards and upload to a HF Dataset repo."""
|
| 218 |
+
import json
|
| 219 |
+
import shutil
|
| 220 |
+
from huggingface_hub import HfApi, create_repo
|
| 221 |
+
|
| 222 |
+
api = HfApi()
|
| 223 |
+
try:
|
| 224 |
+
create_repo(repo_id, repo_type="dataset", exist_ok=True)
|
| 225 |
+
print(f" Dataset repo: https://huggingface.co/datasets/{repo_id}")
|
| 226 |
+
except Exception as e:
|
| 227 |
+
print(f" Repo creation note: {e}")
|
| 228 |
+
|
| 229 |
+
pt_files = sorted(self._cache_dir.glob("*.pt"), key=lambda f: int(f.stem))
|
| 230 |
+
total = len(pt_files)
|
| 231 |
+
print(f" Packing {total} samples into shards...")
|
| 232 |
+
|
| 233 |
+
shard_dir = self._cache_dir / "_shards"
|
| 234 |
+
shard_dir.mkdir(exist_ok=True)
|
| 235 |
+
|
| 236 |
+
shard_idx = 0
|
| 237 |
+
shard_data: list[tuple[torch.Tensor, torch.Tensor]] = []
|
| 238 |
+
|
| 239 |
+
for i, pt_file in enumerate(pt_files):
|
| 240 |
+
mel, tokens = torch.load(pt_file, weights_only=True, map_location="cpu")
|
| 241 |
+
shard_data.append((mel, tokens))
|
| 242 |
+
|
| 243 |
+
if len(shard_data) >= samples_per_shard or i == total - 1:
|
| 244 |
+
shard_path = shard_dir / f"shard_{shard_idx:04d}.pt"
|
| 245 |
+
torch.save(shard_data, shard_path)
|
| 246 |
+
size_mb = shard_path.stat().st_size / 1e6
|
| 247 |
+
print(f" Shard {shard_idx}: {len(shard_data)} samples ({size_mb:.0f} MB)")
|
| 248 |
+
shard_data = []
|
| 249 |
+
shard_idx += 1
|
| 250 |
+
|
| 251 |
+
metadata = {
|
| 252 |
+
"total_samples": total,
|
| 253 |
+
"num_shards": shard_idx,
|
| 254 |
+
"samples_per_shard": samples_per_shard,
|
| 255 |
+
}
|
| 256 |
+
with open(shard_dir / "metadata.json", "w") as f:
|
| 257 |
+
json.dump(metadata, f, indent=2)
|
| 258 |
+
|
| 259 |
+
print(f" Uploading {shard_idx} shards to {repo_id}...")
|
| 260 |
+
api.upload_folder(
|
| 261 |
+
folder_path=str(shard_dir),
|
| 262 |
+
repo_id=repo_id,
|
| 263 |
+
repo_type="dataset",
|
| 264 |
+
commit_message=f"Upload mel cache: {total} samples in {shard_idx} shards",
|
| 265 |
+
)
|
| 266 |
+
print(f" Upload complete!")
|
| 267 |
+
shutil.rmtree(shard_dir)
|
src/repositories/dataset_repository.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Dataset Repository — external data access for HuggingFace audio datasets.
|
| 3 |
+
|
| 4 |
+
Handles fetching from HF Hub, schema resolution, streaming, and
|
| 5 |
+
mel spectrogram extraction. Supports concurrent mel transforms via
|
| 6 |
+
ProcessPoolExecutor for faster caching on multi-core machines.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
import os
|
| 12 |
+
import re
|
| 13 |
+
from concurrent.futures import ProcessPoolExecutor
|
| 14 |
+
from pathlib import Path
|
| 15 |
+
from typing import Iterator
|
| 16 |
+
|
| 17 |
+
import torch
|
| 18 |
+
import torchaudio.transforms as T
|
| 19 |
+
import yaml
|
| 20 |
+
|
| 21 |
+
from src.models.config import GrassmannConfig
|
| 22 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _process_raw_sample(args: tuple) -> tuple[torch.Tensor, torch.Tensor] | None:
|
| 26 |
+
"""Process a single (waveform_array, sr, text, needs_normalize) in a worker.
|
| 27 |
+
|
| 28 |
+
Top-level function so it's picklable by ProcessPoolExecutor.
|
| 29 |
+
Returns (mel, tokens) or None if invalid.
|
| 30 |
+
"""
|
| 31 |
+
waveform_array, sr, text, needs_normalize, config_dict = args
|
| 32 |
+
|
| 33 |
+
# Reconstruct transforms in worker (not picklable across processes)
|
| 34 |
+
target_sr = config_dict["sample_rate"]
|
| 35 |
+
mel_transform = T.MelSpectrogram(
|
| 36 |
+
sample_rate=target_sr, n_fft=config_dict["n_fft"],
|
| 37 |
+
hop_length=config_dict["hop_length"], n_mels=config_dict["n_mels"],
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
try:
|
| 41 |
+
waveform = torch.tensor(waveform_array, dtype=torch.float32)
|
| 42 |
+
if sr != target_sr:
|
| 43 |
+
waveform = T.Resample(sr, target_sr)(waveform)
|
| 44 |
+
if waveform.dim() == 1:
|
| 45 |
+
waveform = waveform.unsqueeze(0)
|
| 46 |
+
|
| 47 |
+
mel = mel_transform(waveform)
|
| 48 |
+
mel = torch.log(mel.clamp(min=1e-9)).squeeze(0)
|
| 49 |
+
mean = mel.mean()
|
| 50 |
+
std = mel.std().clamp(min=1e-9)
|
| 51 |
+
mel = (mel - mean) / std
|
| 52 |
+
|
| 53 |
+
if needs_normalize:
|
| 54 |
+
text = normalize_transcript(text)
|
| 55 |
+
|
| 56 |
+
# Inline tokenization (CharTokenizer is simple enough)
|
| 57 |
+
vocab = {c: i + 1 for i, c in enumerate("abcdefghijklmnopqrstuvwxyz '")}
|
| 58 |
+
tokens = torch.tensor([vocab.get(c, 0) for c in text.lower()], dtype=torch.long)
|
| 59 |
+
if len(tokens) == 0:
|
| 60 |
+
return None
|
| 61 |
+
|
| 62 |
+
return mel, tokens
|
| 63 |
+
except Exception:
|
| 64 |
+
return None
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _load_recipes_config(config_path: str = "recipes.yaml") -> dict:
|
| 68 |
+
"""Load dataset schemas and recipes from YAML config."""
|
| 69 |
+
path = Path(config_path)
|
| 70 |
+
if not path.exists():
|
| 71 |
+
return {"schemas": {}, "recipes": {}}
|
| 72 |
+
with open(path) as f:
|
| 73 |
+
return yaml.safe_load(f)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def normalize_transcript(text: str) -> str:
|
| 77 |
+
"""Normalize transcript to lowercase ASCII with only letters, spaces, apostrophes."""
|
| 78 |
+
text = text.lower().strip()
|
| 79 |
+
text = re.sub(r"[^\w\s']", "", text)
|
| 80 |
+
text = re.sub(r"\s+", " ", text)
|
| 81 |
+
return text.strip()
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
class DatasetRepository:
|
| 85 |
+
"""Fetches audio+text samples from HuggingFace Hub datasets.
|
| 86 |
+
|
| 87 |
+
Resolves column schemas automatically from recipes.yaml or known defaults.
|
| 88 |
+
Streams samples as (mel, tokens) pairs — the caller decides where they go
|
| 89 |
+
(memory, disk cache, etc.).
|
| 90 |
+
"""
|
| 91 |
+
|
| 92 |
+
def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer,
|
| 93 |
+
recipes_path: str = "recipes.yaml"):
|
| 94 |
+
self.config = config
|
| 95 |
+
self.tokenizer = tokenizer
|
| 96 |
+
self._recipes_config = _load_recipes_config(recipes_path)
|
| 97 |
+
self._schemas: dict[str, dict[str, str]] = self._recipes_config.get("schemas", {})
|
| 98 |
+
self._recipes: dict[str, list[dict]] = self._recipes_config.get("recipes", {})
|
| 99 |
+
self._mel_transform = T.MelSpectrogram(
|
| 100 |
+
sample_rate=config.sample_rate, n_fft=config.n_fft,
|
| 101 |
+
hop_length=config.hop_length, n_mels=config.n_mels,
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
def resolve_columns(self, dataset_name: str,
|
| 105 |
+
audio_col: str | None = None,
|
| 106 |
+
text_col: str | None = None) -> tuple[str, str]:
|
| 107 |
+
"""Resolve audio and text column names for a dataset."""
|
| 108 |
+
if audio_col and text_col:
|
| 109 |
+
return audio_col, text_col
|
| 110 |
+
for known_name, schema in self._schemas.items():
|
| 111 |
+
if known_name in dataset_name:
|
| 112 |
+
return audio_col or schema["audio"], text_col or schema["text"]
|
| 113 |
+
return audio_col or "audio", text_col or "text"
|
| 114 |
+
|
| 115 |
+
def get_recipe(self, name: str) -> list[dict]:
|
| 116 |
+
"""Get a named recipe from config."""
|
| 117 |
+
if name not in self._recipes:
|
| 118 |
+
available = list(self._recipes.keys())
|
| 119 |
+
raise ValueError(f"Unknown recipe '{name}'. Available: {available}")
|
| 120 |
+
return self._recipes[name]
|
| 121 |
+
|
| 122 |
+
@property
|
| 123 |
+
def available_recipes(self) -> list[str]:
|
| 124 |
+
return list(self._recipes.keys())
|
| 125 |
+
|
| 126 |
+
def process_sample(self, sample: dict, audio_col: str,
|
| 127 |
+
text_col: str) -> tuple[torch.Tensor, str]:
|
| 128 |
+
"""Extract mel spectrogram and text from a single HF sample."""
|
| 129 |
+
audio_data = sample[audio_col]
|
| 130 |
+
waveform = torch.tensor(audio_data["array"], dtype=torch.float32)
|
| 131 |
+
sr = audio_data["sampling_rate"]
|
| 132 |
+
|
| 133 |
+
if sr != self.config.sample_rate:
|
| 134 |
+
resampler = T.Resample(sr, self.config.sample_rate)
|
| 135 |
+
waveform = resampler(waveform)
|
| 136 |
+
|
| 137 |
+
if waveform.dim() == 1:
|
| 138 |
+
waveform = waveform.unsqueeze(0)
|
| 139 |
+
|
| 140 |
+
mel = self._mel_transform(waveform)
|
| 141 |
+
mel = torch.log(mel.clamp(min=1e-9))
|
| 142 |
+
mel = mel.squeeze(0)
|
| 143 |
+
|
| 144 |
+
mean = mel.mean()
|
| 145 |
+
std = mel.std().clamp(min=1e-9)
|
| 146 |
+
mel = (mel - mean) / std
|
| 147 |
+
|
| 148 |
+
return mel, sample[text_col]
|
| 149 |
+
|
| 150 |
+
def stream(self, dataset_name: str, split: str,
|
| 151 |
+
config_name: str | None = None,
|
| 152 |
+
max_samples: int | None = None,
|
| 153 |
+
audio_col: str | None = None,
|
| 154 |
+
text_col: str | None = None,
|
| 155 |
+
workers: int | None = None) -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
|
| 156 |
+
"""Stream (mel, tokens) pairs from a HF dataset.
|
| 157 |
+
|
| 158 |
+
Uses ProcessPoolExecutor to compute mel spectrograms concurrently.
|
| 159 |
+
Workers defaults to CPU count. Set workers=0 for sequential mode.
|
| 160 |
+
"""
|
| 161 |
+
from datasets import load_dataset
|
| 162 |
+
|
| 163 |
+
a_col, t_col = self.resolve_columns(dataset_name, audio_col, text_col)
|
| 164 |
+
needs_normalize = "librispeech" not in dataset_name.lower()
|
| 165 |
+
n_workers = workers if workers is not None else min(os.cpu_count() or 1, 8)
|
| 166 |
+
|
| 167 |
+
ds = load_dataset(dataset_name, config_name, split=split, streaming=True)
|
| 168 |
+
count = 0
|
| 169 |
+
|
| 170 |
+
# Config dict for worker processes (can't pickle MelSpectrogram)
|
| 171 |
+
config_dict = {
|
| 172 |
+
"sample_rate": self.config.sample_rate,
|
| 173 |
+
"n_fft": self.config.n_fft,
|
| 174 |
+
"hop_length": self.config.hop_length,
|
| 175 |
+
"n_mels": self.config.n_mels,
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
# Sequential fallback
|
| 179 |
+
if n_workers < 2:
|
| 180 |
+
yield from self._stream_sequential(
|
| 181 |
+
ds, a_col, t_col, needs_normalize, max_samples)
|
| 182 |
+
return
|
| 183 |
+
|
| 184 |
+
# Concurrent: prefetch batches of raw samples, fan out mel transforms
|
| 185 |
+
batch_size = n_workers * 4 # keep workers fed
|
| 186 |
+
raw_batch: list[tuple] = []
|
| 187 |
+
|
| 188 |
+
for sample in ds:
|
| 189 |
+
if max_samples and count >= max_samples:
|
| 190 |
+
break
|
| 191 |
+
|
| 192 |
+
try:
|
| 193 |
+
raw_text = sample[t_col].strip()
|
| 194 |
+
except (KeyError, AttributeError):
|
| 195 |
+
continue
|
| 196 |
+
|
| 197 |
+
text = normalize_transcript(raw_text) if needs_normalize else raw_text
|
| 198 |
+
if len(text) == 0:
|
| 199 |
+
continue
|
| 200 |
+
|
| 201 |
+
try:
|
| 202 |
+
audio_array = sample[a_col]["array"]
|
| 203 |
+
except (KeyError, TypeError):
|
| 204 |
+
continue
|
| 205 |
+
|
| 206 |
+
if len(audio_array) < 1600 or len(audio_array) > 240000:
|
| 207 |
+
continue
|
| 208 |
+
|
| 209 |
+
sr = sample[a_col]["sampling_rate"]
|
| 210 |
+
raw_batch.append((audio_array, sr, text, needs_normalize, config_dict))
|
| 211 |
+
|
| 212 |
+
if len(raw_batch) >= batch_size:
|
| 213 |
+
for result in self._process_batch(raw_batch, n_workers):
|
| 214 |
+
if max_samples and count >= max_samples:
|
| 215 |
+
break
|
| 216 |
+
yield result
|
| 217 |
+
count += 1
|
| 218 |
+
raw_batch = []
|
| 219 |
+
|
| 220 |
+
# Drain remaining batch
|
| 221 |
+
if raw_batch and (not max_samples or count < max_samples):
|
| 222 |
+
for result in self._process_batch(raw_batch, n_workers):
|
| 223 |
+
if max_samples and count >= max_samples:
|
| 224 |
+
break
|
| 225 |
+
yield result
|
| 226 |
+
count += 1
|
| 227 |
+
|
| 228 |
+
del ds
|
| 229 |
+
|
| 230 |
+
def _process_batch(self, raw_batch: list[tuple],
|
| 231 |
+
n_workers: int) -> list[tuple[torch.Tensor, torch.Tensor]]:
|
| 232 |
+
"""Process a batch of raw samples concurrently."""
|
| 233 |
+
results = []
|
| 234 |
+
with ProcessPoolExecutor(max_workers=n_workers) as pool:
|
| 235 |
+
for result in pool.map(_process_raw_sample, raw_batch):
|
| 236 |
+
if result is not None:
|
| 237 |
+
results.append(result)
|
| 238 |
+
return results
|
| 239 |
+
|
| 240 |
+
def _stream_sequential(self, ds, a_col: str, t_col: str,
|
| 241 |
+
needs_normalize: bool,
|
| 242 |
+
max_samples: int | None) -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
|
| 243 |
+
"""Sequential fallback for stream()."""
|
| 244 |
+
count = 0
|
| 245 |
+
for sample in ds:
|
| 246 |
+
if max_samples and count >= max_samples:
|
| 247 |
+
break
|
| 248 |
+
|
| 249 |
+
try:
|
| 250 |
+
raw_text = sample[t_col].strip()
|
| 251 |
+
except (KeyError, AttributeError):
|
| 252 |
+
continue
|
| 253 |
+
|
| 254 |
+
text = normalize_transcript(raw_text) if needs_normalize else raw_text
|
| 255 |
+
if len(text) == 0:
|
| 256 |
+
continue
|
| 257 |
+
|
| 258 |
+
try:
|
| 259 |
+
audio_array = sample[a_col]["array"]
|
| 260 |
+
except (KeyError, TypeError):
|
| 261 |
+
continue
|
| 262 |
+
|
| 263 |
+
if len(audio_array) < 1600 or len(audio_array) > 240000:
|
| 264 |
+
continue
|
| 265 |
+
|
| 266 |
+
try:
|
| 267 |
+
mel, _ = self.process_sample(sample, a_col, t_col)
|
| 268 |
+
except Exception:
|
| 269 |
+
continue
|
| 270 |
+
|
| 271 |
+
if needs_normalize:
|
| 272 |
+
text = normalize_transcript(raw_text)
|
| 273 |
+
tokens = self.tokenizer.encode(text)
|
| 274 |
+
if len(tokens) == 0:
|
| 275 |
+
continue
|
| 276 |
+
|
| 277 |
+
yield mel, tokens
|
| 278 |
+
count += 1
|
| 279 |
+
|
| 280 |
+
del ds
|
| 281 |
+
|
| 282 |
+
def fetch_full(self, dataset_name: str, split: str,
|
| 283 |
+
config_name: str | None = None):
|
| 284 |
+
"""Download a full (non-streaming) HF dataset split. Returns the HF dataset object."""
|
| 285 |
+
from datasets import load_dataset
|
| 286 |
+
return load_dataset(dataset_name, config_name, split=split)
|
src/services/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .audio_service import AudioService
|
| 2 |
+
from .tokenizer_service import CharTokenizer
|
| 3 |
+
from .training_service import TrainingService
|
src/services/audio_service.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
import torchaudio
|
| 3 |
+
import torchaudio.transforms as T
|
| 4 |
+
|
| 5 |
+
from src.interfaces import IAudioProcessor
|
| 6 |
+
from src.models.config import GrassmannConfig
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AudioService(IAudioProcessor):
|
| 10 |
+
"""Handles audio loading and mel spectrogram extraction."""
|
| 11 |
+
|
| 12 |
+
def __init__(self, config: GrassmannConfig):
|
| 13 |
+
self.config = config
|
| 14 |
+
self.mel_transform = T.MelSpectrogram(
|
| 15 |
+
sample_rate=config.sample_rate,
|
| 16 |
+
n_fft=config.n_fft,
|
| 17 |
+
hop_length=config.hop_length,
|
| 18 |
+
n_mels=config.n_mels,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
def load(self, path: str) -> torch.Tensor:
|
| 22 |
+
waveform, sr = torchaudio.load(path)
|
| 23 |
+
# Resample if needed
|
| 24 |
+
if sr != self.config.sample_rate:
|
| 25 |
+
resampler = T.Resample(sr, self.config.sample_rate)
|
| 26 |
+
waveform = resampler(waveform)
|
| 27 |
+
# Convert to mono
|
| 28 |
+
if waveform.shape[0] > 1:
|
| 29 |
+
waveform = waveform.mean(dim=0, keepdim=True)
|
| 30 |
+
return waveform.squeeze(0)
|
| 31 |
+
|
| 32 |
+
def extract_features(self, waveform: torch.Tensor) -> torch.Tensor:
|
| 33 |
+
if waveform.dim() == 1:
|
| 34 |
+
waveform = waveform.unsqueeze(0)
|
| 35 |
+
mel = self.mel_transform(waveform) # (1, n_mels, time)
|
| 36 |
+
# Log mel
|
| 37 |
+
mel = torch.log(mel.clamp(min=1e-9))
|
| 38 |
+
return mel.squeeze(0) # (n_mels, time)
|
| 39 |
+
|
| 40 |
+
def normalize(self, features: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
mean = features.mean()
|
| 42 |
+
std = features.std().clamp(min=1e-9)
|
| 43 |
+
return (features - mean) / std
|
src/services/dataset_service.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Dataset Service — orchestrates data loading, splitting, and DataLoader creation.
|
| 3 |
+
|
| 4 |
+
Delegates external data access to repositories:
|
| 5 |
+
- DatasetRepository: HF Hub fetching, schema resolution, streaming
|
| 6 |
+
- CacheRepository: disk cache read/write, HF shard upload/download
|
| 7 |
+
|
| 8 |
+
This service handles: train/val/test splitting, PyTorch Dataset wrappers,
|
| 9 |
+
DataLoader creation, and collation.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
from typing import Self
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
import torchaudio.transforms as T
|
| 18 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 19 |
+
from torch.utils.data import DataLoader, Dataset
|
| 20 |
+
|
| 21 |
+
from src.models.config import GrassmannConfig
|
| 22 |
+
from src.repositories.cache_repository import CacheRepository
|
| 23 |
+
from src.repositories.dataset_repository import DatasetRepository
|
| 24 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# -- PyTorch Dataset wrappers ---------------------------------------------
|
| 28 |
+
|
| 29 |
+
class MaterializedAudioDataset(Dataset):
|
| 30 |
+
"""Pre-materialized dataset -- samples already in memory as (mel, tokens) pairs."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, samples: list[tuple[torch.Tensor, torch.Tensor]]):
|
| 33 |
+
self.samples = samples
|
| 34 |
+
|
| 35 |
+
def __len__(self) -> int:
|
| 36 |
+
return len(self.samples)
|
| 37 |
+
|
| 38 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 39 |
+
return self.samples[idx]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class DiskCachedDataset(Dataset):
|
| 43 |
+
"""Dataset backed by individual .pt files on disk."""
|
| 44 |
+
|
| 45 |
+
def __init__(self, cache: CacheRepository, indices: list[int]):
|
| 46 |
+
self._cache = cache
|
| 47 |
+
self._indices = indices
|
| 48 |
+
|
| 49 |
+
def __len__(self) -> int:
|
| 50 |
+
return len(self._indices)
|
| 51 |
+
|
| 52 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 53 |
+
return self._cache.load(self._indices[idx])
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
class HFAudioDataset(Dataset):
|
| 57 |
+
"""Wraps a (non-streaming) HuggingFace dataset split into a PyTorch Dataset."""
|
| 58 |
+
|
| 59 |
+
def __init__(self, hf_dataset, repo: DatasetRepository,
|
| 60 |
+
audio_col: str, text_col: str):
|
| 61 |
+
self.hf_dataset = hf_dataset
|
| 62 |
+
self._repo = repo
|
| 63 |
+
self.audio_col = audio_col
|
| 64 |
+
self.text_col = text_col
|
| 65 |
+
|
| 66 |
+
def __len__(self) -> int:
|
| 67 |
+
return len(self.hf_dataset)
|
| 68 |
+
|
| 69 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 70 |
+
sample = self.hf_dataset[idx]
|
| 71 |
+
mel, text = self._repo.process_sample(sample, self.audio_col, self.text_col)
|
| 72 |
+
tokens = self._repo.tokenizer.encode(text)
|
| 73 |
+
return mel, tokens
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def collate_fn(
|
| 77 |
+
batch: list[tuple[torch.Tensor, torch.Tensor]],
|
| 78 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 79 |
+
"""Pad mels and tokens to batch-max lengths."""
|
| 80 |
+
mels, tokens = zip(*batch)
|
| 81 |
+
mel_lengths = torch.tensor([m.shape[-1] for m in mels], dtype=torch.long)
|
| 82 |
+
max_mel_len = mel_lengths.max().item()
|
| 83 |
+
padded_mels = []
|
| 84 |
+
for m in mels:
|
| 85 |
+
pad_amount = max_mel_len - m.shape[-1]
|
| 86 |
+
padded_mels.append(torch.nn.functional.pad(m, (0, pad_amount)))
|
| 87 |
+
mel_batch = torch.stack(padded_mels)
|
| 88 |
+
token_lengths = torch.tensor([len(t) for t in tokens], dtype=torch.long)
|
| 89 |
+
token_batch = pad_sequence(tokens, batch_first=True, padding_value=0)
|
| 90 |
+
return mel_batch, token_batch, mel_lengths, token_lengths
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# -- Service ---------------------------------------------------------------
|
| 94 |
+
|
| 95 |
+
class DatasetService:
|
| 96 |
+
"""Orchestrates data loading, splitting, and DataLoader creation.
|
| 97 |
+
|
| 98 |
+
Usage:
|
| 99 |
+
repo = DatasetRepository(config, tokenizer)
|
| 100 |
+
cache = CacheRepository("checkpoints/mel_cache")
|
| 101 |
+
svc = DatasetService(repo, cache)
|
| 102 |
+
|
| 103 |
+
# From HF Hub cache:
|
| 104 |
+
svc.load_from_hub_cache("tisaconundrum/grassmann-mel-cache")
|
| 105 |
+
svc.split(max_samples=28539)
|
| 106 |
+
|
| 107 |
+
# From recipe:
|
| 108 |
+
svc.load_recipe("1750h")
|
| 109 |
+
svc.split()
|
| 110 |
+
|
| 111 |
+
# Direct HF download:
|
| 112 |
+
svc.load_dataset("openslr/librispeech_asr", split="train.clean.100")
|
| 113 |
+
svc.split()
|
| 114 |
+
|
| 115 |
+
train_loader = svc.get_train_loader(batch_size=32)
|
| 116 |
+
"""
|
| 117 |
+
|
| 118 |
+
def __init__(self, repo: DatasetRepository, cache: CacheRepository):
|
| 119 |
+
self._repo = repo
|
| 120 |
+
self._cache = cache
|
| 121 |
+
self._hf_train = None
|
| 122 |
+
self._hf_val = None
|
| 123 |
+
self._hf_test = None
|
| 124 |
+
self._materialized: list[tuple[torch.Tensor, torch.Tensor]] | None = None
|
| 125 |
+
self._use_cache = False
|
| 126 |
+
self._audio_col = "audio"
|
| 127 |
+
self._text_col = "text"
|
| 128 |
+
self._train_ds: Dataset | None = None
|
| 129 |
+
self._val_ds: Dataset | None = None
|
| 130 |
+
self._test_ds: Dataset | None = None
|
| 131 |
+
|
| 132 |
+
def load_from_hub_cache(self, repo_id: str) -> Self:
|
| 133 |
+
"""Download pre-cached mel shards from a HF Dataset repo."""
|
| 134 |
+
self._cache.download_from_hub(repo_id)
|
| 135 |
+
self._use_cache = True
|
| 136 |
+
return self
|
| 137 |
+
|
| 138 |
+
def load_recipe(self, recipe_name: str,
|
| 139 |
+
max_samples_per_source: int | None = None,
|
| 140 |
+
upload_repo: str | None = None,
|
| 141 |
+
samples_per_shard: int = 50000,
|
| 142 |
+
workers: int = 0) -> Self:
|
| 143 |
+
"""Stream multiple datasets per recipe and cache to disk.
|
| 144 |
+
|
| 145 |
+
When upload_repo is set, shards are uploaded incrementally to HF Hub
|
| 146 |
+
as they fill up — progress survives container restarts.
|
| 147 |
+
"""
|
| 148 |
+
sources = self._repo.get_recipe(recipe_name)
|
| 149 |
+
print(f" Loading recipe '{recipe_name}': {len(sources)} sources")
|
| 150 |
+
if upload_repo:
|
| 151 |
+
print(f" Incremental upload to: {upload_repo}")
|
| 152 |
+
for src in sources:
|
| 153 |
+
cfg_str = f" ({src['config']})" if src.get('config') else ""
|
| 154 |
+
print(f" - {src['name']} / {src['split']}{cfg_str}")
|
| 155 |
+
|
| 156 |
+
for src in sources:
|
| 157 |
+
src_cap = src.get("max_samples") or max_samples_per_source
|
| 158 |
+
stream = self._repo.stream(
|
| 159 |
+
src["name"], src["split"],
|
| 160 |
+
config_name=src.get("config"),
|
| 161 |
+
max_samples=src_cap,
|
| 162 |
+
workers=workers,
|
| 163 |
+
)
|
| 164 |
+
label = f"{src['name']}/{src['split']}"
|
| 165 |
+
count = self._cache.save_stream(
|
| 166 |
+
stream, source_label=label,
|
| 167 |
+
repo_id=upload_repo, samples_per_shard=samples_per_shard,
|
| 168 |
+
)
|
| 169 |
+
print(f" {label}: {count} samples")
|
| 170 |
+
|
| 171 |
+
self._use_cache = True
|
| 172 |
+
return self
|
| 173 |
+
|
| 174 |
+
def load_dataset(self, dataset_name: str, split: str | None = None,
|
| 175 |
+
config_name: str | None = None,
|
| 176 |
+
max_samples: int | None = None) -> Self:
|
| 177 |
+
"""Load a single dataset -- streaming or full download."""
|
| 178 |
+
self._audio_col, self._text_col = self._repo.resolve_columns(dataset_name)
|
| 179 |
+
print(f" Loading dataset: {dataset_name}")
|
| 180 |
+
print(f" Columns: audio='{self._audio_col}', text='{self._text_col}'")
|
| 181 |
+
|
| 182 |
+
if max_samples is not None:
|
| 183 |
+
stream = self._repo.stream(
|
| 184 |
+
dataset_name, split or "train",
|
| 185 |
+
config_name=config_name, max_samples=max_samples,
|
| 186 |
+
)
|
| 187 |
+
self._materialized = list(stream)
|
| 188 |
+
print(f" Materialized {len(self._materialized)} samples into memory.")
|
| 189 |
+
return self
|
| 190 |
+
|
| 191 |
+
# Full download
|
| 192 |
+
if split:
|
| 193 |
+
self._hf_train = self._repo.fetch_full(dataset_name, split, config_name)
|
| 194 |
+
print(f" Loaded split '{split}': {len(self._hf_train)} samples")
|
| 195 |
+
else:
|
| 196 |
+
try:
|
| 197 |
+
self._hf_train = self._repo.fetch_full(dataset_name, "train", config_name)
|
| 198 |
+
except Exception:
|
| 199 |
+
pass
|
| 200 |
+
try:
|
| 201 |
+
self._hf_val = self._repo.fetch_full(dataset_name, "validation", config_name)
|
| 202 |
+
except Exception:
|
| 203 |
+
pass
|
| 204 |
+
try:
|
| 205 |
+
self._hf_test = self._repo.fetch_full(dataset_name, "test", config_name)
|
| 206 |
+
except Exception:
|
| 207 |
+
pass
|
| 208 |
+
sizes = []
|
| 209 |
+
if self._hf_train: sizes.append(f"train={len(self._hf_train)}")
|
| 210 |
+
if self._hf_val: sizes.append(f"val={len(self._hf_val)}")
|
| 211 |
+
if self._hf_test: sizes.append(f"test={len(self._hf_test)}")
|
| 212 |
+
print(f" Loaded splits: {', '.join(sizes)}")
|
| 213 |
+
return self
|
| 214 |
+
|
| 215 |
+
def clean(self) -> Self:
|
| 216 |
+
"""Filter out empty/invalid samples from full-download datasets."""
|
| 217 |
+
if self._hf_train is not None and hasattr(self._hf_train, "filter"):
|
| 218 |
+
original_len = len(self._hf_train)
|
| 219 |
+
a, t = self._audio_col, self._text_col
|
| 220 |
+
self._hf_train = self._hf_train.filter(
|
| 221 |
+
lambda x: len(x[t].strip()) > 0 and len(x[a]["array"]) > 1600
|
| 222 |
+
)
|
| 223 |
+
print(f" Cleaned: {original_len} -> {len(self._hf_train)} samples")
|
| 224 |
+
return self
|
| 225 |
+
|
| 226 |
+
def preprocess(self) -> Self:
|
| 227 |
+
return self
|
| 228 |
+
|
| 229 |
+
def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1,
|
| 230 |
+
max_samples: int | None = None) -> Self:
|
| 231 |
+
"""Split loaded data into train/val/test."""
|
| 232 |
+
if self._use_cache:
|
| 233 |
+
all_indices = self._cache.indices()
|
| 234 |
+
if max_samples is not None:
|
| 235 |
+
all_indices = all_indices[:max_samples]
|
| 236 |
+
n = len(all_indices)
|
| 237 |
+
n_train = max(1, int(n * train_ratio))
|
| 238 |
+
n_val = max(1, int(n * val_ratio))
|
| 239 |
+
self._train_ds = DiskCachedDataset(self._cache, all_indices[:n_train])
|
| 240 |
+
self._val_ds = DiskCachedDataset(self._cache, all_indices[n_train:n_train + n_val])
|
| 241 |
+
rest = all_indices[n_train + n_val:]
|
| 242 |
+
self._test_ds = DiskCachedDataset(self._cache, rest if rest else all_indices[n_train:n_train + n_val])
|
| 243 |
+
|
| 244 |
+
elif self._materialized is not None:
|
| 245 |
+
samples = self._materialized
|
| 246 |
+
n = len(samples)
|
| 247 |
+
n_train = max(1, int(n * train_ratio))
|
| 248 |
+
n_val = max(1, int(n * val_ratio))
|
| 249 |
+
self._train_ds = MaterializedAudioDataset(samples[:n_train])
|
| 250 |
+
self._val_ds = MaterializedAudioDataset(samples[n_train:n_train + n_val])
|
| 251 |
+
rest = samples[n_train + n_val:]
|
| 252 |
+
self._test_ds = MaterializedAudioDataset(rest if rest else samples[n_train:n_train + n_val])
|
| 253 |
+
|
| 254 |
+
elif self._hf_val is not None:
|
| 255 |
+
self._train_ds = HFAudioDataset(self._hf_train, self._repo, self._audio_col, self._text_col)
|
| 256 |
+
self._val_ds = HFAudioDataset(self._hf_val, self._repo, self._audio_col, self._text_col)
|
| 257 |
+
self._test_ds = HFAudioDataset(self._hf_test, self._repo, self._audio_col, self._text_col) if self._hf_test else self._val_ds
|
| 258 |
+
|
| 259 |
+
elif self._hf_train is not None:
|
| 260 |
+
n = len(self._hf_train)
|
| 261 |
+
n_train = int(n * train_ratio)
|
| 262 |
+
n_val = int(n * val_ratio)
|
| 263 |
+
split_ds = self._hf_train.train_test_split(test_size=(n - n_train) / n, seed=42)
|
| 264 |
+
train_hf = split_ds["train"]
|
| 265 |
+
remaining = split_ds["test"]
|
| 266 |
+
if len(remaining) > 1:
|
| 267 |
+
val_test = remaining.train_test_split(
|
| 268 |
+
test_size=max(1, n - n_train - n_val) / len(remaining), seed=42)
|
| 269 |
+
val_hf, test_hf = val_test["train"], val_test["test"]
|
| 270 |
+
else:
|
| 271 |
+
val_hf = test_hf = remaining
|
| 272 |
+
self._train_ds = HFAudioDataset(train_hf, self._repo, self._audio_col, self._text_col)
|
| 273 |
+
self._val_ds = HFAudioDataset(val_hf, self._repo, self._audio_col, self._text_col)
|
| 274 |
+
self._test_ds = HFAudioDataset(test_hf, self._repo, self._audio_col, self._text_col)
|
| 275 |
+
|
| 276 |
+
if self._train_ds:
|
| 277 |
+
print(f" Splits ready: train={len(self._train_ds)}, "
|
| 278 |
+
f"val={len(self._val_ds) if self._val_ds else 0}, "
|
| 279 |
+
f"test={len(self._test_ds) if self._test_ds else 0}")
|
| 280 |
+
return self
|
| 281 |
+
|
| 282 |
+
def get_train_loader(self, batch_size: int) -> DataLoader:
|
| 283 |
+
assert self._train_ds is not None, "Call load_*() and split() first"
|
| 284 |
+
return DataLoader(self._train_ds, batch_size=batch_size, shuffle=True,
|
| 285 |
+
collate_fn=collate_fn, num_workers=0)
|
| 286 |
+
|
| 287 |
+
def get_val_loader(self, batch_size: int) -> DataLoader:
|
| 288 |
+
assert self._val_ds is not None, "Call load_*() and split() first"
|
| 289 |
+
return DataLoader(self._val_ds, batch_size=batch_size, shuffle=False,
|
| 290 |
+
collate_fn=collate_fn, num_workers=0)
|
| 291 |
+
|
| 292 |
+
def get_test_loader(self, batch_size: int) -> DataLoader:
|
| 293 |
+
assert self._test_ds is not None, "Call load_*() and split() first"
|
| 294 |
+
return DataLoader(self._test_ds, batch_size=batch_size, shuffle=False,
|
| 295 |
+
collate_fn=collate_fn, num_workers=0)
|
src/services/eval_service.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Evaluation Service.
|
| 3 |
+
|
| 4 |
+
WER/CER evaluation on LibriSpeech (or other HF datasets).
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import time
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
import torchaudio
|
| 13 |
+
import torchaudio.transforms
|
| 14 |
+
|
| 15 |
+
from src.models.config import GrassmannConfig
|
| 16 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
_mel_transform = None
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def make_mel(waveform: torch.Tensor, sample_rate: int, config: GrassmannConfig) -> torch.Tensor:
|
| 23 |
+
"""Convert waveform to normalized mel spectrogram (matches training pipeline)."""
|
| 24 |
+
global _mel_transform
|
| 25 |
+
if _mel_transform is None:
|
| 26 |
+
_mel_transform = torchaudio.transforms.MelSpectrogram(
|
| 27 |
+
sample_rate=config.sample_rate, n_fft=config.n_fft,
|
| 28 |
+
hop_length=config.hop_length, n_mels=config.n_mels,
|
| 29 |
+
)
|
| 30 |
+
|
| 31 |
+
if sample_rate != config.sample_rate:
|
| 32 |
+
waveform = torchaudio.functional.resample(waveform, sample_rate, config.sample_rate)
|
| 33 |
+
if waveform.dim() == 1:
|
| 34 |
+
waveform = waveform.unsqueeze(0)
|
| 35 |
+
|
| 36 |
+
mel = _mel_transform(waveform)
|
| 37 |
+
mel = torch.log(mel.clamp(min=1e-9)).squeeze(0)
|
| 38 |
+
|
| 39 |
+
mean = mel.mean()
|
| 40 |
+
std = mel.std().clamp(min=1e-9)
|
| 41 |
+
mel = (mel - mean) / std
|
| 42 |
+
|
| 43 |
+
return mel.unsqueeze(0)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
@torch.no_grad()
|
| 47 |
+
def greedy_decode(model, mel: torch.Tensor, tokenizer: CharTokenizer) -> str:
|
| 48 |
+
log_probs = model(mel)
|
| 49 |
+
preds = log_probs.argmax(dim=-1).squeeze(1)
|
| 50 |
+
return tokenizer.decode(preds)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def evaluate_wer(
|
| 54 |
+
model,
|
| 55 |
+
config: GrassmannConfig,
|
| 56 |
+
tokenizer: CharTokenizer,
|
| 57 |
+
split: str = "test.clean",
|
| 58 |
+
max_samples: int | None = None,
|
| 59 |
+
show_errors: int = 10,
|
| 60 |
+
) -> dict:
|
| 61 |
+
"""Run WER/CER evaluation on a LibriSpeech split.
|
| 62 |
+
|
| 63 |
+
Returns dict with wer, cer, num_samples, elapsed_seconds.
|
| 64 |
+
"""
|
| 65 |
+
from jiwer import wer as compute_wer, cer as compute_cer
|
| 66 |
+
from datasets import load_dataset
|
| 67 |
+
|
| 68 |
+
model.eval()
|
| 69 |
+
print(f" Loading LibriSpeech {split}...")
|
| 70 |
+
ds = load_dataset("openslr/librispeech_asr", split=split, streaming=True)
|
| 71 |
+
|
| 72 |
+
references, hypotheses, errors = [], [], []
|
| 73 |
+
start = time.time()
|
| 74 |
+
|
| 75 |
+
for i, sample in enumerate(ds):
|
| 76 |
+
if max_samples and i >= max_samples:
|
| 77 |
+
break
|
| 78 |
+
|
| 79 |
+
audio = sample["audio"]
|
| 80 |
+
waveform = torch.tensor(audio["array"], dtype=torch.float32).unsqueeze(0)
|
| 81 |
+
sr = audio["sampling_rate"]
|
| 82 |
+
ref = sample["text"].lower().strip()
|
| 83 |
+
|
| 84 |
+
mel = make_mel(waveform, sr, config)
|
| 85 |
+
hyp = greedy_decode(model, mel, tokenizer).strip()
|
| 86 |
+
|
| 87 |
+
references.append(ref)
|
| 88 |
+
hypotheses.append(hyp)
|
| 89 |
+
|
| 90 |
+
sample_wer = compute_wer(ref, hyp) if ref else 0.0
|
| 91 |
+
errors.append((sample_wer, ref, hyp))
|
| 92 |
+
|
| 93 |
+
if (i + 1) % 100 == 0:
|
| 94 |
+
elapsed = time.time() - start
|
| 95 |
+
current_wer = compute_wer(references, hypotheses)
|
| 96 |
+
print(f" {i+1} samples | WER: {current_wer:.2%} | {elapsed:.0f}s", flush=True)
|
| 97 |
+
|
| 98 |
+
total_time = time.time() - start
|
| 99 |
+
total_wer = compute_wer(references, hypotheses)
|
| 100 |
+
total_cer = compute_cer(references, hypotheses)
|
| 101 |
+
|
| 102 |
+
print(f"\n{'=' * 60}")
|
| 103 |
+
print(f" LibriSpeech {split} — Greedy CTC Decode")
|
| 104 |
+
print(f"{'=' * 60}")
|
| 105 |
+
print(f" Samples: {len(references)}")
|
| 106 |
+
print(f" WER: {total_wer:.2%}")
|
| 107 |
+
print(f" CER: {total_cer:.2%}")
|
| 108 |
+
print(f" Time: {total_time:.1f}s ({total_time/len(references):.2f}s/sample)")
|
| 109 |
+
print(f"{'=' * 60}")
|
| 110 |
+
|
| 111 |
+
if show_errors > 0:
|
| 112 |
+
errors.sort(key=lambda x: -x[0])
|
| 113 |
+
print(f"\n Top {show_errors} worst transcriptions:")
|
| 114 |
+
for j, (w, ref, hyp) in enumerate(errors[:show_errors]):
|
| 115 |
+
print(f"\n [{j+1}] WER: {w:.0%}")
|
| 116 |
+
print(f" GT: '{ref}'")
|
| 117 |
+
print(f" PRED: '{hyp}'")
|
| 118 |
+
|
| 119 |
+
return {
|
| 120 |
+
"wer": total_wer,
|
| 121 |
+
"cer": total_cer,
|
| 122 |
+
"num_samples": len(references),
|
| 123 |
+
"elapsed_seconds": total_time,
|
| 124 |
+
}
|
src/services/export_service.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ONNX Export Service.
|
| 3 |
+
|
| 4 |
+
Handles exporting Grassmann models to ONNX format for cross-platform inference.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from src.models.config import GrassmannConfig
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class EncoderWrapper(torch.nn.Module):
|
| 17 |
+
def __init__(self, encoder):
|
| 18 |
+
super().__init__()
|
| 19 |
+
self.encoder = encoder
|
| 20 |
+
|
| 21 |
+
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
| 22 |
+
return self.encoder(mel)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class DecoderStepWrapper(torch.nn.Module):
|
| 26 |
+
def __init__(self, decoder):
|
| 27 |
+
super().__init__()
|
| 28 |
+
self.decoder = decoder
|
| 29 |
+
|
| 30 |
+
def forward(self, tokens: torch.Tensor, encoder_out: torch.Tensor) -> torch.Tensor:
|
| 31 |
+
logits = self.decoder(tokens, encoder_out)
|
| 32 |
+
return logits[:, -1:, :]
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class CTCModelWrapper(torch.nn.Module):
|
| 36 |
+
def __init__(self, model):
|
| 37 |
+
super().__init__()
|
| 38 |
+
self.frontend = model.frontend
|
| 39 |
+
self.pos_enc = model.pos_enc
|
| 40 |
+
self.layers = model.layers
|
| 41 |
+
self.output_proj = model.output_proj
|
| 42 |
+
|
| 43 |
+
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
| 44 |
+
x = self.frontend(mel)
|
| 45 |
+
x = self.pos_enc(x)
|
| 46 |
+
for layer in self.layers:
|
| 47 |
+
x = layer(x)
|
| 48 |
+
logits = self.output_proj(x)
|
| 49 |
+
return logits.log_softmax(dim=-1)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def export_seq2seq(model, config: GrassmannConfig, output_dir: str, opset: int = 18) -> None:
|
| 53 |
+
model = model.cpu()
|
| 54 |
+
model.eval()
|
| 55 |
+
out = Path(output_dir)
|
| 56 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 57 |
+
|
| 58 |
+
print(" Exporting encoder.onnx...")
|
| 59 |
+
encoder_wrapper = EncoderWrapper(model.encoder)
|
| 60 |
+
encoder_wrapper.eval()
|
| 61 |
+
dummy_mel = torch.randn(1, config.n_mels, 200)
|
| 62 |
+
torch.onnx.export(
|
| 63 |
+
encoder_wrapper, (dummy_mel,), str(out / "encoder.onnx"),
|
| 64 |
+
input_names=["mel"], output_names=["encoder_out"],
|
| 65 |
+
dynamic_axes={"mel": {0: "batch", 2: "time"}, "encoder_out": {0: "batch", 1: "enc_time"}},
|
| 66 |
+
opset_version=opset,
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
print(" Exporting decoder_step.onnx...")
|
| 70 |
+
decoder_wrapper = DecoderStepWrapper(model.decoder)
|
| 71 |
+
decoder_wrapper.eval()
|
| 72 |
+
dummy_tokens = torch.tensor([[0, 5, 12]], dtype=torch.long)
|
| 73 |
+
dummy_enc_out = torch.randn(1, 50, config.d_model)
|
| 74 |
+
torch.onnx.export(
|
| 75 |
+
decoder_wrapper, (dummy_tokens, dummy_enc_out), str(out / "decoder_step.onnx"),
|
| 76 |
+
input_names=["tokens", "encoder_out"], output_names=["next_logits"],
|
| 77 |
+
dynamic_axes={"tokens": {0: "batch", 1: "seq_len"}, "encoder_out": {0: "batch", 1: "enc_time"}, "next_logits": {0: "batch"}},
|
| 78 |
+
opset_version=opset,
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def export_ctc(model, config: GrassmannConfig, output_dir: str, opset: int = 18) -> None:
|
| 83 |
+
model = model.cpu()
|
| 84 |
+
model.eval()
|
| 85 |
+
out = Path(output_dir)
|
| 86 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 87 |
+
|
| 88 |
+
print(" Exporting model.onnx (CTC)...")
|
| 89 |
+
wrapper = CTCModelWrapper(model)
|
| 90 |
+
wrapper.eval()
|
| 91 |
+
dummy_mel = torch.randn(1, config.n_mels, 200)
|
| 92 |
+
torch.onnx.export(
|
| 93 |
+
wrapper, (dummy_mel,), str(out / "model.onnx"),
|
| 94 |
+
input_names=["mel"], output_names=["log_probs"],
|
| 95 |
+
dynamic_axes={"mel": {0: "batch", 2: "time"}, "log_probs": {0: "batch", 1: "time"}},
|
| 96 |
+
opset_version=opset,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def verify_onnx(output_dir: str, config: GrassmannConfig, is_seq2seq: bool) -> None:
|
| 101 |
+
try:
|
| 102 |
+
import onnxruntime as ort
|
| 103 |
+
except ImportError:
|
| 104 |
+
print(" Install onnxruntime to verify: pip install onnxruntime")
|
| 105 |
+
return
|
| 106 |
+
|
| 107 |
+
import numpy as np
|
| 108 |
+
out = Path(output_dir)
|
| 109 |
+
|
| 110 |
+
if is_seq2seq:
|
| 111 |
+
print(" Verifying encoder.onnx...")
|
| 112 |
+
enc_session = ort.InferenceSession(str(out / "encoder.onnx"))
|
| 113 |
+
mel = np.random.randn(1, config.n_mels, 200).astype(np.float32)
|
| 114 |
+
enc_out = enc_session.run(None, {"mel": mel})[0]
|
| 115 |
+
print(f" Encoder output shape: {enc_out.shape}")
|
| 116 |
+
|
| 117 |
+
print(" Verifying decoder_step.onnx...")
|
| 118 |
+
dec_session = ort.InferenceSession(str(out / "decoder_step.onnx"))
|
| 119 |
+
tokens = np.array([[0, 5, 12]], dtype=np.int64)
|
| 120 |
+
logits = dec_session.run(None, {"tokens": tokens, "encoder_out": enc_out})[0]
|
| 121 |
+
print(f" Decoder logits shape: {logits.shape}")
|
| 122 |
+
else:
|
| 123 |
+
print(" Verifying model.onnx (CTC)...")
|
| 124 |
+
session = ort.InferenceSession(str(out / "model.onnx"))
|
| 125 |
+
mel = np.random.randn(1, config.n_mels, 200).astype(np.float32)
|
| 126 |
+
log_probs = session.run(None, {"mel": mel})[0]
|
| 127 |
+
print(f" Output shape: {log_probs.shape}")
|
| 128 |
+
|
| 129 |
+
print(" ONNX verification passed!")
|
src/services/hf_dataset_service.py
ADDED
|
@@ -0,0 +1,657 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
HuggingFace Dataset Service.
|
| 3 |
+
|
| 4 |
+
Loads any HuggingFace audio+text dataset (LibriSpeech, Common Voice, FLEURS, etc.)
|
| 5 |
+
and wraps it into PyTorch DataLoaders for the Grassmann transcription pipeline.
|
| 6 |
+
|
| 7 |
+
Uses streaming mode by default when max_samples is set, so we only download
|
| 8 |
+
the rows we actually need instead of multi-GB parquet shards.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from __future__ import annotations
|
| 12 |
+
|
| 13 |
+
from typing import Self
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torchaudio.transforms as T
|
| 17 |
+
from torch.nn.utils.rnn import pad_sequence
|
| 18 |
+
from torch.utils.data import DataLoader, Dataset
|
| 19 |
+
|
| 20 |
+
from src.interfaces import IDataset
|
| 21 |
+
from src.models.config import GrassmannConfig
|
| 22 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
import re
|
| 26 |
+
|
| 27 |
+
KNOWN_SCHEMAS: dict[str, tuple[str, str]] = {
|
| 28 |
+
"openslr/librispeech_asr": ("audio", "text"),
|
| 29 |
+
"mozilla-foundation/common_voice": ("audio", "sentence"),
|
| 30 |
+
"google/fleurs": ("audio", "transcription"),
|
| 31 |
+
"speechcolab/gigaspeech": ("audio", "text"),
|
| 32 |
+
"facebook/voxpopuli": ("audio", "normalized_text"),
|
| 33 |
+
"MLCommons/peoples_speech": ("audio", "text"),
|
| 34 |
+
"espnet/yodas": ("audio", "text"),
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
# Predefined multi-dataset recipes
|
| 38 |
+
DATASET_RECIPES: dict[str, list[dict]] = {
|
| 39 |
+
"librispeech-960": [
|
| 40 |
+
{"name": "openslr/librispeech_asr", "split": "train.clean.100"},
|
| 41 |
+
{"name": "openslr/librispeech_asr", "split": "train.clean.360"},
|
| 42 |
+
{"name": "openslr/librispeech_asr", "split": "train.other.500"},
|
| 43 |
+
],
|
| 44 |
+
"1750h": [
|
| 45 |
+
{"name": "openslr/librispeech_asr", "split": "train.clean.100"},
|
| 46 |
+
{"name": "openslr/librispeech_asr", "split": "train.clean.360"},
|
| 47 |
+
{"name": "openslr/librispeech_asr", "split": "train.other.500"},
|
| 48 |
+
{"name": "facebook/voxpopuli", "config": "en", "split": "train"},
|
| 49 |
+
{"name": "MLCommons/peoples_speech", "config": "clean", "split": "train",
|
| 50 |
+
"max_samples": 30000}, # ~250h subset of 30K hours
|
| 51 |
+
],
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def normalize_transcript(text: str) -> str:
|
| 56 |
+
"""Normalize transcript to lowercase ASCII with only letters, spaces, apostrophes."""
|
| 57 |
+
text = text.lower().strip()
|
| 58 |
+
text = re.sub(r"[^\w\s']", "", text)
|
| 59 |
+
text = re.sub(r"\s+", " ", text)
|
| 60 |
+
return text.strip()
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _resolve_columns(dataset_name: str, audio_col: str | None, text_col: str | None) -> tuple[str, str]:
|
| 64 |
+
if audio_col and text_col:
|
| 65 |
+
return audio_col, text_col
|
| 66 |
+
for known_name, (a, t) in KNOWN_SCHEMAS.items():
|
| 67 |
+
if known_name in dataset_name:
|
| 68 |
+
return audio_col or a, text_col or t
|
| 69 |
+
return audio_col or "audio", text_col or "text"
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _process_sample(
|
| 73 |
+
sample: dict,
|
| 74 |
+
audio_col: str,
|
| 75 |
+
text_col: str,
|
| 76 |
+
mel_transform: T.MelSpectrogram,
|
| 77 |
+
target_sr: int,
|
| 78 |
+
) -> tuple[torch.Tensor, str]:
|
| 79 |
+
"""Extract mel spectrogram and text from a single HF sample."""
|
| 80 |
+
audio_data = sample[audio_col]
|
| 81 |
+
waveform = torch.tensor(audio_data["array"], dtype=torch.float32)
|
| 82 |
+
sr = audio_data["sampling_rate"]
|
| 83 |
+
|
| 84 |
+
if sr != target_sr:
|
| 85 |
+
resampler = T.Resample(sr, target_sr)
|
| 86 |
+
waveform = resampler(waveform)
|
| 87 |
+
|
| 88 |
+
if waveform.dim() == 1:
|
| 89 |
+
waveform = waveform.unsqueeze(0)
|
| 90 |
+
|
| 91 |
+
mel = mel_transform(waveform)
|
| 92 |
+
mel = torch.log(mel.clamp(min=1e-9))
|
| 93 |
+
mel = mel.squeeze(0)
|
| 94 |
+
|
| 95 |
+
mean = mel.mean()
|
| 96 |
+
std = mel.std().clamp(min=1e-9)
|
| 97 |
+
mel = (mel - mean) / std
|
| 98 |
+
|
| 99 |
+
return mel, sample[text_col]
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class MaterializedAudioDataset(Dataset):
|
| 103 |
+
"""Pre-materialized dataset -- samples already in memory as (mel, tokens) pairs."""
|
| 104 |
+
|
| 105 |
+
def __init__(self, samples: list[tuple[torch.Tensor, torch.Tensor]]):
|
| 106 |
+
self.samples = samples
|
| 107 |
+
|
| 108 |
+
def __len__(self) -> int:
|
| 109 |
+
return len(self.samples)
|
| 110 |
+
|
| 111 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 112 |
+
return self.samples[idx]
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class DiskCachedDataset(Dataset):
|
| 116 |
+
"""Dataset backed by individual .pt files on disk. Low RAM usage.
|
| 117 |
+
|
| 118 |
+
Uses a single index file to avoid thousands of small file opens.
|
| 119 |
+
On first access, builds an in-memory index mapping idx -> (file_offset).
|
| 120 |
+
"""
|
| 121 |
+
|
| 122 |
+
def __init__(self, cache_dir: str, indices: list[int] | None = None):
|
| 123 |
+
from pathlib import Path
|
| 124 |
+
self._cache_dir = Path(cache_dir)
|
| 125 |
+
# If indices provided, use subset; otherwise use all files
|
| 126 |
+
if indices is not None:
|
| 127 |
+
self._indices = indices
|
| 128 |
+
else:
|
| 129 |
+
self._indices = sorted(
|
| 130 |
+
int(f.stem) for f in self._cache_dir.glob("*.pt")
|
| 131 |
+
)
|
| 132 |
+
|
| 133 |
+
def __len__(self) -> int:
|
| 134 |
+
return len(self._indices)
|
| 135 |
+
|
| 136 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 137 |
+
file_idx = self._indices[idx]
|
| 138 |
+
mel, tokens = torch.load(
|
| 139 |
+
self._cache_dir / f"{file_idx}.pt", weights_only=True,
|
| 140 |
+
map_location="cpu",
|
| 141 |
+
)
|
| 142 |
+
# Clone to detach from the file-backed storage, then delete originals
|
| 143 |
+
mel_out, tok_out = mel.contiguous().clone(), tokens.contiguous().clone()
|
| 144 |
+
del mel, tokens
|
| 145 |
+
return mel_out, tok_out
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class HFAudioDataset(Dataset):
|
| 149 |
+
"""Wraps a (non-streaming) HuggingFace dataset split into a PyTorch Dataset."""
|
| 150 |
+
|
| 151 |
+
def __init__(self, hf_dataset, audio_col: str, text_col: str,
|
| 152 |
+
tokenizer: CharTokenizer, config: GrassmannConfig):
|
| 153 |
+
self.hf_dataset = hf_dataset
|
| 154 |
+
self.audio_col = audio_col
|
| 155 |
+
self.text_col = text_col
|
| 156 |
+
self.tokenizer = tokenizer
|
| 157 |
+
self.mel_transform = T.MelSpectrogram(
|
| 158 |
+
sample_rate=config.sample_rate, n_fft=config.n_fft,
|
| 159 |
+
hop_length=config.hop_length, n_mels=config.n_mels,
|
| 160 |
+
)
|
| 161 |
+
self.target_sr = config.sample_rate
|
| 162 |
+
|
| 163 |
+
def __len__(self) -> int:
|
| 164 |
+
return len(self.hf_dataset)
|
| 165 |
+
|
| 166 |
+
def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
|
| 167 |
+
sample = self.hf_dataset[idx]
|
| 168 |
+
mel, text = _process_sample(
|
| 169 |
+
sample, self.audio_col, self.text_col, self.mel_transform, self.target_sr
|
| 170 |
+
)
|
| 171 |
+
tokens = self.tokenizer.encode(text)
|
| 172 |
+
return mel, tokens
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def hf_collate_fn(
|
| 176 |
+
batch: list[tuple[torch.Tensor, torch.Tensor]],
|
| 177 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 178 |
+
mels, tokens = zip(*batch)
|
| 179 |
+
mel_lengths = torch.tensor([m.shape[-1] for m in mels], dtype=torch.long)
|
| 180 |
+
max_mel_len = mel_lengths.max().item()
|
| 181 |
+
padded_mels = []
|
| 182 |
+
for m in mels:
|
| 183 |
+
pad_amount = max_mel_len - m.shape[-1]
|
| 184 |
+
padded_mels.append(torch.nn.functional.pad(m, (0, pad_amount)))
|
| 185 |
+
mel_batch = torch.stack(padded_mels)
|
| 186 |
+
token_lengths = torch.tensor([len(t) for t in tokens], dtype=torch.long)
|
| 187 |
+
token_batch = pad_sequence(tokens, batch_first=True, padding_value=0)
|
| 188 |
+
return mel_batch, token_batch, mel_lengths, token_lengths
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _stream_and_materialize(
|
| 192 |
+
dataset_name: str,
|
| 193 |
+
config_name: str | None,
|
| 194 |
+
split: str,
|
| 195 |
+
max_samples: int,
|
| 196 |
+
audio_col: str,
|
| 197 |
+
text_col: str,
|
| 198 |
+
tokenizer: CharTokenizer,
|
| 199 |
+
config: GrassmannConfig,
|
| 200 |
+
cache_dir: str | None = None,
|
| 201 |
+
) -> list[tuple[torch.Tensor, torch.Tensor]] | str:
|
| 202 |
+
"""Stream a HF dataset and save processed samples.
|
| 203 |
+
|
| 204 |
+
If cache_dir is set, saves each (mel, tokens) pair as a .pt file on disk
|
| 205 |
+
and returns the cache_dir path (low RAM). Otherwise materializes into a list
|
| 206 |
+
in memory (original behavior).
|
| 207 |
+
"""
|
| 208 |
+
from pathlib import Path
|
| 209 |
+
from datasets import load_dataset
|
| 210 |
+
|
| 211 |
+
# Check if cache already exists
|
| 212 |
+
if cache_dir:
|
| 213 |
+
cache_path = Path(cache_dir)
|
| 214 |
+
existing = list(cache_path.glob("*.pt"))
|
| 215 |
+
if existing:
|
| 216 |
+
print(f" Found {len(existing)} cached samples in {cache_dir}, skipping stream.")
|
| 217 |
+
return cache_dir
|
| 218 |
+
cache_path.mkdir(parents=True, exist_ok=True)
|
| 219 |
+
|
| 220 |
+
print(f" Streaming {max_samples} samples from '{split}'...")
|
| 221 |
+
ds = load_dataset(dataset_name, config_name, split=split, streaming=True)
|
| 222 |
+
|
| 223 |
+
mel_transform = T.MelSpectrogram(
|
| 224 |
+
sample_rate=config.sample_rate, n_fft=config.n_fft,
|
| 225 |
+
hop_length=config.hop_length, n_mels=config.n_mels,
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
import gc
|
| 229 |
+
|
| 230 |
+
samples: list[tuple[torch.Tensor, torch.Tensor]] = []
|
| 231 |
+
saved_count = 0
|
| 232 |
+
for i, sample in enumerate(ds):
|
| 233 |
+
if i >= max_samples:
|
| 234 |
+
break
|
| 235 |
+
text = sample[text_col].strip()
|
| 236 |
+
audio_array = sample[audio_col]["array"]
|
| 237 |
+
# Skip empty, too-short (<0.1s), or too-long (>15s at 16kHz = 240000 samples) clips
|
| 238 |
+
# Long clips cause massive padding waste in collate_fn
|
| 239 |
+
if len(text) == 0 or len(audio_array) < 1600 or len(audio_array) > 240000:
|
| 240 |
+
del sample, audio_array
|
| 241 |
+
continue
|
| 242 |
+
mel, text = _process_sample(sample, audio_col, text_col, mel_transform, config.sample_rate)
|
| 243 |
+
tokens = tokenizer.encode(text)
|
| 244 |
+
if cache_dir:
|
| 245 |
+
torch.save((mel, tokens), Path(cache_dir) / f"{saved_count}.pt")
|
| 246 |
+
saved_count += 1
|
| 247 |
+
del mel, tokens
|
| 248 |
+
else:
|
| 249 |
+
samples.append((mel, tokens))
|
| 250 |
+
del sample, audio_array
|
| 251 |
+
if (i + 1) % 10 == 0:
|
| 252 |
+
print(f" Processed {i + 1}/{max_samples} samples...")
|
| 253 |
+
# Periodic GC during caching to prevent RAM buildup from librosa/datasets buffers
|
| 254 |
+
if cache_dir and (i + 1) % 500 == 0:
|
| 255 |
+
gc.collect()
|
| 256 |
+
|
| 257 |
+
# Explicitly release the streaming dataset and its internal buffers
|
| 258 |
+
del ds
|
| 259 |
+
gc.collect()
|
| 260 |
+
|
| 261 |
+
if cache_dir:
|
| 262 |
+
print(f" Cached {saved_count} samples to disk at {cache_dir}.")
|
| 263 |
+
return cache_dir
|
| 264 |
+
else:
|
| 265 |
+
print(f" Materialized {len(samples)} samples into memory.")
|
| 266 |
+
return samples
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _stream_multi_datasets(
|
| 270 |
+
sources: list[dict],
|
| 271 |
+
tokenizer: CharTokenizer,
|
| 272 |
+
config: GrassmannConfig,
|
| 273 |
+
cache_dir: str,
|
| 274 |
+
max_samples_per_source: int | None = None,
|
| 275 |
+
) -> str:
|
| 276 |
+
"""Stream multiple HF datasets and cache all samples to disk.
|
| 277 |
+
|
| 278 |
+
Each source dict has: name, split, and optionally config.
|
| 279 |
+
Non-LibriSpeech datasets get normalize_transcript() applied.
|
| 280 |
+
|
| 281 |
+
Returns the cache_dir path.
|
| 282 |
+
"""
|
| 283 |
+
from pathlib import Path
|
| 284 |
+
from datasets import load_dataset
|
| 285 |
+
import gc
|
| 286 |
+
|
| 287 |
+
cache_path = Path(cache_dir)
|
| 288 |
+
existing = list(cache_path.glob("*.pt"))
|
| 289 |
+
if existing:
|
| 290 |
+
print(f" Found {len(existing)} cached samples in {cache_dir}, skipping stream.")
|
| 291 |
+
return cache_dir
|
| 292 |
+
cache_path.mkdir(parents=True, exist_ok=True)
|
| 293 |
+
|
| 294 |
+
mel_transform = T.MelSpectrogram(
|
| 295 |
+
sample_rate=config.sample_rate, n_fft=config.n_fft,
|
| 296 |
+
hop_length=config.hop_length, n_mels=config.n_mels,
|
| 297 |
+
)
|
| 298 |
+
|
| 299 |
+
saved_count = 0
|
| 300 |
+
for src in sources:
|
| 301 |
+
name = src["name"]
|
| 302 |
+
split = src["split"]
|
| 303 |
+
src_config = src.get("config")
|
| 304 |
+
audio_col, text_col = _resolve_columns(name, None, None)
|
| 305 |
+
needs_normalize = "librispeech" not in name.lower()
|
| 306 |
+
|
| 307 |
+
config_str = f" ({src_config})" if src_config else ""
|
| 308 |
+
norm_str = " (normalizing)" if needs_normalize else ""
|
| 309 |
+
print(f"\n [{saved_count} cached] Streaming {name} / {split}{config_str}{norm_str}...")
|
| 310 |
+
|
| 311 |
+
ds = load_dataset(name, src_config, split=split, streaming=True)
|
| 312 |
+
src_count = 0
|
| 313 |
+
# Per-source cap: use source-level override, then global cap, then unlimited
|
| 314 |
+
src_cap = src.get("max_samples") or max_samples_per_source
|
| 315 |
+
|
| 316 |
+
for i, sample in enumerate(ds):
|
| 317 |
+
if src_cap and src_count >= src_cap:
|
| 318 |
+
break
|
| 319 |
+
|
| 320 |
+
try:
|
| 321 |
+
raw_text = sample[text_col].strip()
|
| 322 |
+
except (KeyError, AttributeError):
|
| 323 |
+
continue
|
| 324 |
+
|
| 325 |
+
text = normalize_transcript(raw_text) if needs_normalize else raw_text
|
| 326 |
+
if len(text) == 0:
|
| 327 |
+
continue
|
| 328 |
+
|
| 329 |
+
try:
|
| 330 |
+
audio_array = sample[audio_col]["array"]
|
| 331 |
+
except (KeyError, TypeError):
|
| 332 |
+
continue
|
| 333 |
+
|
| 334 |
+
# Skip too-short (<0.1s) or too-long (>15s) clips
|
| 335 |
+
if len(audio_array) < 1600 or len(audio_array) > 240000:
|
| 336 |
+
continue
|
| 337 |
+
|
| 338 |
+
try:
|
| 339 |
+
mel, _ = _process_sample(sample, audio_col, text_col, mel_transform, config.sample_rate)
|
| 340 |
+
except Exception:
|
| 341 |
+
continue
|
| 342 |
+
|
| 343 |
+
# Re-normalize the text (process_sample returns raw text)
|
| 344 |
+
if needs_normalize:
|
| 345 |
+
text = normalize_transcript(raw_text)
|
| 346 |
+
tokens = tokenizer.encode(text)
|
| 347 |
+
|
| 348 |
+
if len(tokens) == 0:
|
| 349 |
+
continue
|
| 350 |
+
|
| 351 |
+
torch.save((mel, tokens), cache_path / f"{saved_count}.pt")
|
| 352 |
+
saved_count += 1
|
| 353 |
+
src_count += 1
|
| 354 |
+
|
| 355 |
+
del mel, tokens, sample, audio_array
|
| 356 |
+
if src_count % 1000 == 0:
|
| 357 |
+
print(f" {name}: {src_count} samples cached ({saved_count} total)...")
|
| 358 |
+
gc.collect()
|
| 359 |
+
|
| 360 |
+
del ds
|
| 361 |
+
gc.collect()
|
| 362 |
+
print(f" {name}/{split}: {src_count} samples")
|
| 363 |
+
|
| 364 |
+
print(f"\n Total: {saved_count} samples cached to {cache_dir}")
|
| 365 |
+
return cache_dir
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
class HFDatasetService(IDataset):
|
| 369 |
+
"""Loads audio datasets from HuggingFace Hub.
|
| 370 |
+
|
| 371 |
+
Usage:
|
| 372 |
+
_dataset = HFDatasetService(config, tokenizer)
|
| 373 |
+
_dataset.load("openslr/librispeech_asr", split="train.clean.100",
|
| 374 |
+
max_samples=100)
|
| 375 |
+
.clean().preprocess().split()
|
| 376 |
+
|
| 377 |
+
# Multi-dataset mode:
|
| 378 |
+
_dataset.load_multi("1750h", cache_dir="checkpoints/mel_cache")
|
| 379 |
+
.clean().preprocess().split()
|
| 380 |
+
"""
|
| 381 |
+
|
| 382 |
+
def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer):
|
| 383 |
+
self.config = config
|
| 384 |
+
self.tokenizer = tokenizer
|
| 385 |
+
self.dataset_name: str = ""
|
| 386 |
+
self.audio_col: str = "audio"
|
| 387 |
+
self.text_col: str = "text"
|
| 388 |
+
self._hf_train = None
|
| 389 |
+
self._hf_val = None
|
| 390 |
+
self._hf_test = None
|
| 391 |
+
self._materialized: list[tuple[torch.Tensor, torch.Tensor]] | None = None
|
| 392 |
+
self._cache_dir: str | None = None
|
| 393 |
+
self._train_ds: Dataset | None = None
|
| 394 |
+
self._val_ds: Dataset | None = None
|
| 395 |
+
self._test_ds: Dataset | None = None
|
| 396 |
+
|
| 397 |
+
def load(self, dataset_name: str, config_name: str | None = None,
|
| 398 |
+
split: str | None = None, audio_col: str | None = None,
|
| 399 |
+
text_col: str | None = None, streaming: bool = False,
|
| 400 |
+
max_samples: int | None = None,
|
| 401 |
+
cache_dir: str | None = None) -> Self:
|
| 402 |
+
from datasets import load_dataset
|
| 403 |
+
|
| 404 |
+
self.dataset_name = dataset_name
|
| 405 |
+
self.audio_col, self.text_col = _resolve_columns(dataset_name, audio_col, text_col)
|
| 406 |
+
print(f" Loading dataset: {dataset_name}")
|
| 407 |
+
print(f" Columns: audio='{self.audio_col}', text='{self.text_col}'")
|
| 408 |
+
|
| 409 |
+
# When max_samples is set, stream to avoid downloading everything
|
| 410 |
+
if max_samples is not None:
|
| 411 |
+
train_split = split or "train"
|
| 412 |
+
result = _stream_and_materialize(
|
| 413 |
+
dataset_name, config_name, train_split, max_samples,
|
| 414 |
+
self.audio_col, self.text_col, self.tokenizer, self.config,
|
| 415 |
+
cache_dir=cache_dir,
|
| 416 |
+
)
|
| 417 |
+
if isinstance(result, str):
|
| 418 |
+
self._cache_dir = result
|
| 419 |
+
else:
|
| 420 |
+
self._materialized = result
|
| 421 |
+
return self
|
| 422 |
+
|
| 423 |
+
# Full download mode
|
| 424 |
+
if split:
|
| 425 |
+
ds = load_dataset(dataset_name, config_name, split=split)
|
| 426 |
+
self._hf_train = ds
|
| 427 |
+
print(f" Loaded split '{split}': {len(ds)} samples")
|
| 428 |
+
else:
|
| 429 |
+
try:
|
| 430 |
+
self._hf_train = load_dataset(dataset_name, config_name, split="train")
|
| 431 |
+
except Exception:
|
| 432 |
+
pass
|
| 433 |
+
try:
|
| 434 |
+
self._hf_val = load_dataset(dataset_name, config_name, split="validation")
|
| 435 |
+
except Exception:
|
| 436 |
+
pass
|
| 437 |
+
try:
|
| 438 |
+
self._hf_test = load_dataset(dataset_name, config_name, split="test")
|
| 439 |
+
except Exception:
|
| 440 |
+
pass
|
| 441 |
+
sizes = []
|
| 442 |
+
if self._hf_train: sizes.append(f"train={len(self._hf_train)}")
|
| 443 |
+
if self._hf_val: sizes.append(f"val={len(self._hf_val)}")
|
| 444 |
+
if self._hf_test: sizes.append(f"test={len(self._hf_test)}")
|
| 445 |
+
print(f" Loaded splits: {', '.join(sizes)}")
|
| 446 |
+
return self
|
| 447 |
+
|
| 448 |
+
def load_multi(
|
| 449 |
+
self,
|
| 450 |
+
recipe: str | list[dict],
|
| 451 |
+
cache_dir: str,
|
| 452 |
+
max_samples_per_source: int | None = None,
|
| 453 |
+
) -> Self:
|
| 454 |
+
"""Load multiple datasets using a recipe name or explicit source list.
|
| 455 |
+
|
| 456 |
+
Args:
|
| 457 |
+
recipe: Either a key from DATASET_RECIPES (e.g. "1750h", "librispeech-960")
|
| 458 |
+
or a list of dicts with name/split/config keys.
|
| 459 |
+
cache_dir: Directory to cache processed mel+token pairs.
|
| 460 |
+
max_samples_per_source: Cap samples per dataset (for testing).
|
| 461 |
+
"""
|
| 462 |
+
if isinstance(recipe, str):
|
| 463 |
+
if recipe not in DATASET_RECIPES:
|
| 464 |
+
raise ValueError(f"Unknown recipe '{recipe}'. Available: {list(DATASET_RECIPES.keys())}")
|
| 465 |
+
sources = DATASET_RECIPES[recipe]
|
| 466 |
+
print(f" Loading recipe '{recipe}': {len(sources)} sources")
|
| 467 |
+
else:
|
| 468 |
+
sources = recipe
|
| 469 |
+
print(f" Loading {len(sources)} custom sources")
|
| 470 |
+
|
| 471 |
+
for src in sources:
|
| 472 |
+
cfg_str = f" ({src['config']})" if src.get('config') else ""
|
| 473 |
+
print(f" - {src['name']} / {src['split']}{cfg_str}")
|
| 474 |
+
|
| 475 |
+
self._cache_dir = _stream_multi_datasets(
|
| 476 |
+
sources, self.tokenizer, self.config, cache_dir,
|
| 477 |
+
max_samples_per_source=max_samples_per_source,
|
| 478 |
+
)
|
| 479 |
+
return self
|
| 480 |
+
|
| 481 |
+
def load_from_hub_cache(self, repo_id: str, cache_dir: str) -> Self:
|
| 482 |
+
"""Download pre-cached mel shards from a HF Dataset repo and unpack to .pt files.
|
| 483 |
+
|
| 484 |
+
Use this to skip expensive CPU caching when mels have already been
|
| 485 |
+
pre-computed and uploaded via upload_cache().
|
| 486 |
+
"""
|
| 487 |
+
from pathlib import Path
|
| 488 |
+
from huggingface_hub import hf_hub_download, list_repo_files
|
| 489 |
+
|
| 490 |
+
cache_path = Path(cache_dir)
|
| 491 |
+
existing = list(cache_path.glob("*.pt"))
|
| 492 |
+
if existing:
|
| 493 |
+
print(f" Found {len(existing)} cached samples in {cache_dir}, skipping download.")
|
| 494 |
+
self._cache_dir = cache_dir
|
| 495 |
+
return self
|
| 496 |
+
|
| 497 |
+
cache_path.mkdir(parents=True, exist_ok=True)
|
| 498 |
+
print(f" Downloading mel cache from {repo_id}...")
|
| 499 |
+
|
| 500 |
+
files = list_repo_files(repo_id, repo_type="dataset")
|
| 501 |
+
shard_files = sorted(f for f in files if f.startswith("shard_") and f.endswith(".pt"))
|
| 502 |
+
print(f" Found {len(shard_files)} shards")
|
| 503 |
+
|
| 504 |
+
sample_idx = 0
|
| 505 |
+
for shard_name in shard_files:
|
| 506 |
+
local_path = hf_hub_download(repo_id, shard_name, repo_type="dataset")
|
| 507 |
+
shard_data = torch.load(local_path, weights_only=True, map_location="cpu")
|
| 508 |
+
for mel, tokens in shard_data:
|
| 509 |
+
torch.save((mel, tokens), cache_path / f"{sample_idx}.pt")
|
| 510 |
+
sample_idx += 1
|
| 511 |
+
del shard_data
|
| 512 |
+
print(f" Unpacked {shard_name} -> {sample_idx} samples total")
|
| 513 |
+
|
| 514 |
+
print(f" Downloaded and unpacked {sample_idx} samples to {cache_dir}")
|
| 515 |
+
self._cache_dir = cache_dir
|
| 516 |
+
return self
|
| 517 |
+
|
| 518 |
+
def upload_cache(self, repo_id: str) -> None:
|
| 519 |
+
"""Upload the mel cache directory as shards to a HF Dataset repo.
|
| 520 |
+
|
| 521 |
+
Call after load_multi() to persist the cache so GPU training
|
| 522 |
+
can use load_from_hub_cache() instead of re-streaming.
|
| 523 |
+
"""
|
| 524 |
+
import json
|
| 525 |
+
import shutil
|
| 526 |
+
from pathlib import Path
|
| 527 |
+
from huggingface_hub import HfApi, create_repo
|
| 528 |
+
|
| 529 |
+
assert self._cache_dir is not None, "No cache to upload. Call load_multi() first."
|
| 530 |
+
|
| 531 |
+
api = HfApi()
|
| 532 |
+
try:
|
| 533 |
+
create_repo(repo_id, repo_type="dataset", exist_ok=True)
|
| 534 |
+
print(f" Dataset repo: https://huggingface.co/datasets/{repo_id}")
|
| 535 |
+
except Exception as e:
|
| 536 |
+
print(f" Repo creation note: {e}")
|
| 537 |
+
|
| 538 |
+
cache_path = Path(self._cache_dir)
|
| 539 |
+
pt_files = sorted(cache_path.glob("*.pt"), key=lambda f: int(f.stem))
|
| 540 |
+
total = len(pt_files)
|
| 541 |
+
print(f" Packing {total} samples into shards...")
|
| 542 |
+
|
| 543 |
+
shard_dir = cache_path / "_shards"
|
| 544 |
+
shard_dir.mkdir(exist_ok=True)
|
| 545 |
+
|
| 546 |
+
SAMPLES_PER_SHARD = 50000
|
| 547 |
+
shard_idx = 0
|
| 548 |
+
shard_data = []
|
| 549 |
+
|
| 550 |
+
for i, pt_file in enumerate(pt_files):
|
| 551 |
+
mel, tokens = torch.load(pt_file, weights_only=True, map_location="cpu")
|
| 552 |
+
shard_data.append((mel, tokens))
|
| 553 |
+
|
| 554 |
+
if len(shard_data) >= SAMPLES_PER_SHARD or i == total - 1:
|
| 555 |
+
shard_path = shard_dir / f"shard_{shard_idx:04d}.pt"
|
| 556 |
+
torch.save(shard_data, shard_path)
|
| 557 |
+
size_mb = shard_path.stat().st_size / 1e6
|
| 558 |
+
print(f" Shard {shard_idx}: {len(shard_data)} samples ({size_mb:.0f} MB)")
|
| 559 |
+
shard_data = []
|
| 560 |
+
shard_idx += 1
|
| 561 |
+
|
| 562 |
+
metadata = {
|
| 563 |
+
"total_samples": total,
|
| 564 |
+
"num_shards": shard_idx,
|
| 565 |
+
"samples_per_shard": SAMPLES_PER_SHARD,
|
| 566 |
+
}
|
| 567 |
+
with open(shard_dir / "metadata.json", "w") as f:
|
| 568 |
+
json.dump(metadata, f, indent=2)
|
| 569 |
+
|
| 570 |
+
print(f" Uploading {shard_idx} shards to {repo_id}...")
|
| 571 |
+
api.upload_folder(
|
| 572 |
+
folder_path=str(shard_dir),
|
| 573 |
+
repo_id=repo_id,
|
| 574 |
+
repo_type="dataset",
|
| 575 |
+
commit_message=f"Upload mel cache: {total} samples in {shard_idx} shards",
|
| 576 |
+
)
|
| 577 |
+
print(f" Upload complete!")
|
| 578 |
+
shutil.rmtree(shard_dir)
|
| 579 |
+
|
| 580 |
+
def clean(self) -> Self:
|
| 581 |
+
if self._materialized is not None:
|
| 582 |
+
return self # already filtered during streaming
|
| 583 |
+
if self._hf_train is not None and hasattr(self._hf_train, "filter"):
|
| 584 |
+
original_len = len(self._hf_train)
|
| 585 |
+
self._hf_train = self._hf_train.filter(
|
| 586 |
+
lambda x: len(x[self.text_col].strip()) > 0
|
| 587 |
+
and len(x[self.audio_col]["array"]) > 1600
|
| 588 |
+
)
|
| 589 |
+
print(f" Cleaned: {original_len} -> {len(self._hf_train)} samples")
|
| 590 |
+
return self
|
| 591 |
+
|
| 592 |
+
def preprocess(self) -> Self:
|
| 593 |
+
return self
|
| 594 |
+
|
| 595 |
+
def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1,
|
| 596 |
+
max_samples: int | None = None) -> Self:
|
| 597 |
+
if self._cache_dir is not None:
|
| 598 |
+
# Disk-cached mode: split by file indices
|
| 599 |
+
from pathlib import Path
|
| 600 |
+
all_indices = sorted(
|
| 601 |
+
int(f.stem) for f in Path(self._cache_dir).glob("*.pt")
|
| 602 |
+
)
|
| 603 |
+
if max_samples is not None:
|
| 604 |
+
all_indices = all_indices[:max_samples]
|
| 605 |
+
n = len(all_indices)
|
| 606 |
+
n_train = max(1, int(n * train_ratio))
|
| 607 |
+
n_val = max(1, int(n * val_ratio))
|
| 608 |
+
self._train_ds = DiskCachedDataset(self._cache_dir, all_indices[:n_train])
|
| 609 |
+
self._val_ds = DiskCachedDataset(self._cache_dir, all_indices[n_train:n_train + n_val])
|
| 610 |
+
rest = all_indices[n_train + n_val:]
|
| 611 |
+
self._test_ds = DiskCachedDataset(self._cache_dir, rest if rest else all_indices[n_train:n_train + n_val])
|
| 612 |
+
elif self._materialized is not None:
|
| 613 |
+
samples = self._materialized
|
| 614 |
+
n = len(samples)
|
| 615 |
+
n_train = max(1, int(n * train_ratio))
|
| 616 |
+
n_val = max(1, int(n * val_ratio))
|
| 617 |
+
self._train_ds = MaterializedAudioDataset(samples[:n_train])
|
| 618 |
+
self._val_ds = MaterializedAudioDataset(samples[n_train:n_train + n_val])
|
| 619 |
+
rest = samples[n_train + n_val:]
|
| 620 |
+
self._test_ds = MaterializedAudioDataset(rest if rest else samples[n_train:n_train + n_val])
|
| 621 |
+
elif self._hf_val is not None:
|
| 622 |
+
self._train_ds = HFAudioDataset(self._hf_train, self.audio_col, self.text_col, self.tokenizer, self.config)
|
| 623 |
+
self._val_ds = HFAudioDataset(self._hf_val, self.audio_col, self.text_col, self.tokenizer, self.config)
|
| 624 |
+
self._test_ds = HFAudioDataset(self._hf_test, self.audio_col, self.text_col, self.tokenizer, self.config) if self._hf_test else self._val_ds
|
| 625 |
+
elif self._hf_train is not None:
|
| 626 |
+
n = len(self._hf_train)
|
| 627 |
+
n_train = int(n * train_ratio)
|
| 628 |
+
n_val = int(n * val_ratio)
|
| 629 |
+
split_ds = self._hf_train.train_test_split(test_size=(n - n_train) / n, seed=42)
|
| 630 |
+
train_hf = split_ds["train"]
|
| 631 |
+
remaining = split_ds["test"]
|
| 632 |
+
if len(remaining) > 1:
|
| 633 |
+
val_test = remaining.train_test_split(test_size=max(1, n - n_train - n_val) / len(remaining), seed=42)
|
| 634 |
+
val_hf, test_hf = val_test["train"], val_test["test"]
|
| 635 |
+
else:
|
| 636 |
+
val_hf = test_hf = remaining
|
| 637 |
+
self._train_ds = HFAudioDataset(train_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
|
| 638 |
+
self._val_ds = HFAudioDataset(val_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
|
| 639 |
+
self._test_ds = HFAudioDataset(test_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
|
| 640 |
+
|
| 641 |
+
if self._train_ds:
|
| 642 |
+
print(f" Splits ready: train={len(self._train_ds)}, "
|
| 643 |
+
f"val={len(self._val_ds) if self._val_ds else 0}, "
|
| 644 |
+
f"test={len(self._test_ds) if self._test_ds else 0}")
|
| 645 |
+
return self
|
| 646 |
+
|
| 647 |
+
def get_train_loader(self, batch_size: int) -> DataLoader:
|
| 648 |
+
assert self._train_ds is not None, "Call load() and split() first"
|
| 649 |
+
return DataLoader(self._train_ds, batch_size=batch_size, shuffle=True, collate_fn=hf_collate_fn, num_workers=0)
|
| 650 |
+
|
| 651 |
+
def get_val_loader(self, batch_size: int) -> DataLoader:
|
| 652 |
+
assert self._val_ds is not None, "Call load() and split() first"
|
| 653 |
+
return DataLoader(self._val_ds, batch_size=batch_size, shuffle=False, collate_fn=hf_collate_fn, num_workers=0)
|
| 654 |
+
|
| 655 |
+
def get_test_loader(self, batch_size: int) -> DataLoader:
|
| 656 |
+
assert self._test_ds is not None, "Call load() and split() first"
|
| 657 |
+
return DataLoader(self._test_ds, batch_size=batch_size, shuffle=False, collate_fn=hf_collate_fn, num_workers=0)
|
src/services/model_hub_service.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Model Hub Service — Save/load models in a self-contained format.
|
| 3 |
+
|
| 4 |
+
Bundles config + weights + tokenizer info into a single directory
|
| 5 |
+
that anyone can download and run inference with.
|
| 6 |
+
|
| 7 |
+
Saved format:
|
| 8 |
+
model_dir/
|
| 9 |
+
config.json # GrassmannConfig as JSON
|
| 10 |
+
model.pt # state_dict weights
|
| 11 |
+
tokenizer_info.json # vocab info for reconstruction
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
from dataclasses import asdict
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import torch
|
| 21 |
+
|
| 22 |
+
from src.models.config import GrassmannConfig
|
| 23 |
+
from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
|
| 24 |
+
from src.models.grassmann_model import GrassmannTranscriptionModel
|
| 25 |
+
from src.services.tokenizer_service import CharTokenizer
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def save_model(
|
| 29 |
+
model: GrassmannSeq2SeqModel | GrassmannTranscriptionModel,
|
| 30 |
+
config: GrassmannConfig,
|
| 31 |
+
save_dir: str,
|
| 32 |
+
) -> None:
|
| 33 |
+
"""Save model + config in a self-contained directory."""
|
| 34 |
+
path = Path(save_dir)
|
| 35 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 36 |
+
|
| 37 |
+
# Save config
|
| 38 |
+
config_dict = asdict(config)
|
| 39 |
+
with open(path / "config.json", "w") as f:
|
| 40 |
+
json.dump(config_dict, f, indent=2)
|
| 41 |
+
|
| 42 |
+
# Save weights
|
| 43 |
+
torch.save(model.state_dict(), path / "model.pt")
|
| 44 |
+
|
| 45 |
+
# Save tokenizer info
|
| 46 |
+
tokenizer = CharTokenizer()
|
| 47 |
+
tok_info = {
|
| 48 |
+
"type": "char",
|
| 49 |
+
"vocab_size": tokenizer.vocab_size,
|
| 50 |
+
"blank_id": tokenizer.blank_id,
|
| 51 |
+
}
|
| 52 |
+
with open(path / "tokenizer_info.json", "w") as f:
|
| 53 |
+
json.dump(tok_info, f, indent=2)
|
| 54 |
+
|
| 55 |
+
print(f" Model saved to {save_dir}/")
|
| 56 |
+
print(f" config.json, model.pt, tokenizer_info.json")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def load_model(
|
| 60 |
+
load_dir: str,
|
| 61 |
+
device: str = "cpu",
|
| 62 |
+
) -> tuple[GrassmannSeq2SeqModel | GrassmannTranscriptionModel, GrassmannConfig, CharTokenizer]:
|
| 63 |
+
"""Load a saved model from directory. Returns (model, config, tokenizer)."""
|
| 64 |
+
path = Path(load_dir)
|
| 65 |
+
|
| 66 |
+
# Load config
|
| 67 |
+
with open(path / "config.json") as f:
|
| 68 |
+
config_dict = json.load(f)
|
| 69 |
+
config = GrassmannConfig(**config_dict)
|
| 70 |
+
|
| 71 |
+
# Build model
|
| 72 |
+
tokenizer = CharTokenizer()
|
| 73 |
+
config.vocab_size = tokenizer.vocab_size
|
| 74 |
+
|
| 75 |
+
if config.use_seq2seq:
|
| 76 |
+
model = GrassmannSeq2SeqModel(config)
|
| 77 |
+
else:
|
| 78 |
+
model = GrassmannTranscriptionModel(config)
|
| 79 |
+
|
| 80 |
+
# Load weights
|
| 81 |
+
state = torch.load(path / "model.pt", map_location=device, weights_only=True)
|
| 82 |
+
model.load_state_dict(state)
|
| 83 |
+
model.to(device)
|
| 84 |
+
model.eval()
|
| 85 |
+
|
| 86 |
+
return model, config, tokenizer
|
src/services/seq2seq_training_service.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Seq2Seq Training Service.
|
| 3 |
+
|
| 4 |
+
Handles teacher-forced training for the Grassmann encoder-decoder model.
|
| 5 |
+
Uses cross-entropy loss instead of CTC, with proper token shifting
|
| 6 |
+
and padding masks.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from dataclasses import dataclass
|
| 12 |
+
|
| 13 |
+
import torch
|
| 14 |
+
import torch.nn as nn
|
| 15 |
+
from torch.utils.data import DataLoader
|
| 16 |
+
|
| 17 |
+
from src.models.config import GrassmannConfig
|
| 18 |
+
from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass
|
| 22 |
+
class TrainMetrics:
|
| 23 |
+
epoch: int
|
| 24 |
+
train_loss: float
|
| 25 |
+
val_loss: float | None = None
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class Seq2SeqTrainingService:
|
| 29 |
+
"""Training loop for the fully attention-free encoder-decoder."""
|
| 30 |
+
|
| 31 |
+
def __init__(
|
| 32 |
+
self,
|
| 33 |
+
model: GrassmannSeq2SeqModel,
|
| 34 |
+
config: GrassmannConfig,
|
| 35 |
+
device: torch.device | None = None,
|
| 36 |
+
):
|
| 37 |
+
self.model = model
|
| 38 |
+
self.config = config
|
| 39 |
+
|
| 40 |
+
if device is not None:
|
| 41 |
+
self.device = device
|
| 42 |
+
elif torch.cuda.is_available():
|
| 43 |
+
self.device = torch.device("cuda")
|
| 44 |
+
else:
|
| 45 |
+
self.device = torch.device("cpu")
|
| 46 |
+
|
| 47 |
+
self.model.to(self.device)
|
| 48 |
+
|
| 49 |
+
self.optimizer = torch.optim.AdamW(
|
| 50 |
+
model.parameters(),
|
| 51 |
+
lr=config.learning_rate,
|
| 52 |
+
weight_decay=config.weight_decay,
|
| 53 |
+
)
|
| 54 |
+
self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
|
| 55 |
+
|
| 56 |
+
# Cross-entropy loss, ignoring padding (index 0)
|
| 57 |
+
self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
|
| 58 |
+
self.history: list[TrainMetrics] = []
|
| 59 |
+
|
| 60 |
+
def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
|
| 61 |
+
from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
|
| 62 |
+
warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
|
| 63 |
+
cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
|
| 64 |
+
self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
|
| 65 |
+
|
| 66 |
+
def train_epoch(self, train_loader: DataLoader) -> float:
|
| 67 |
+
self.model.train()
|
| 68 |
+
total_loss = 0.0
|
| 69 |
+
n_batches = 0
|
| 70 |
+
total_batches = len(train_loader)
|
| 71 |
+
|
| 72 |
+
for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
|
| 73 |
+
mels = mels.to(self.device)
|
| 74 |
+
tokens = tokens.to(self.device)
|
| 75 |
+
|
| 76 |
+
self.optimizer.zero_grad()
|
| 77 |
+
|
| 78 |
+
# Teacher-forced forward pass
|
| 79 |
+
logits = self.model(mels, tokens) # (B, T_dec, vocab)
|
| 80 |
+
|
| 81 |
+
# Reshape for cross-entropy: (B*T, vocab) vs (B*T,)
|
| 82 |
+
B, T, V = logits.shape
|
| 83 |
+
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
|
| 84 |
+
|
| 85 |
+
if torch.isfinite(loss):
|
| 86 |
+
loss.backward()
|
| 87 |
+
torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
|
| 88 |
+
self.optimizer.step()
|
| 89 |
+
total_loss += loss.item()
|
| 90 |
+
n_batches += 1
|
| 91 |
+
|
| 92 |
+
# Log every 10 batches for CI visibility
|
| 93 |
+
if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
|
| 94 |
+
avg = total_loss / max(n_batches, 1)
|
| 95 |
+
print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
|
| 96 |
+
|
| 97 |
+
return total_loss / max(n_batches, 1)
|
| 98 |
+
|
| 99 |
+
@torch.no_grad()
|
| 100 |
+
def validate(self, val_loader: DataLoader) -> float:
|
| 101 |
+
self.model.eval()
|
| 102 |
+
total_loss = 0.0
|
| 103 |
+
n_batches = 0
|
| 104 |
+
|
| 105 |
+
for mels, tokens, mel_lengths, token_lengths in val_loader:
|
| 106 |
+
mels = mels.to(self.device)
|
| 107 |
+
tokens = tokens.to(self.device)
|
| 108 |
+
|
| 109 |
+
logits = self.model(mels, tokens)
|
| 110 |
+
|
| 111 |
+
B, T, V = logits.shape
|
| 112 |
+
loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
|
| 113 |
+
|
| 114 |
+
if torch.isfinite(loss):
|
| 115 |
+
total_loss += loss.item()
|
| 116 |
+
n_batches += 1
|
| 117 |
+
|
| 118 |
+
return total_loss / max(n_batches, 1)
|
| 119 |
+
|
| 120 |
+
def train_loop(
|
| 121 |
+
self,
|
| 122 |
+
train_loader: DataLoader,
|
| 123 |
+
val_loader: DataLoader | None,
|
| 124 |
+
epochs: int,
|
| 125 |
+
callback: object | None = None,
|
| 126 |
+
) -> list[TrainMetrics]:
|
| 127 |
+
for epoch in range(1, epochs + 1):
|
| 128 |
+
train_loss = self.train_epoch(train_loader)
|
| 129 |
+
val_loss = self.validate(val_loader) if val_loader else None
|
| 130 |
+
|
| 131 |
+
metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
|
| 132 |
+
self.history.append(metrics)
|
| 133 |
+
|
| 134 |
+
if callback and hasattr(callback, "on_epoch_end"):
|
| 135 |
+
callback.on_epoch_end(metrics)
|
| 136 |
+
|
| 137 |
+
if self.scheduler is not None:
|
| 138 |
+
self.scheduler.step()
|
| 139 |
+
|
| 140 |
+
return self.history
|
| 141 |
+
|
| 142 |
+
@torch.no_grad()
|
| 143 |
+
def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
|
| 144 |
+
"""Autoregressive greedy decoding for a single mel spectrogram."""
|
| 145 |
+
self.model.eval()
|
| 146 |
+
if mel.dim() == 2:
|
| 147 |
+
mel = mel.unsqueeze(0)
|
| 148 |
+
mel = mel.to(self.device)
|
| 149 |
+
return self.model.greedy_decode(mel, max_len=200)
|
src/services/tokenizer_service.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
from src.interfaces import ITokenizer
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class CharTokenizer(ITokenizer):
|
| 11 |
+
"""
|
| 12 |
+
Character-level tokenizer for transcription.
|
| 13 |
+
|
| 14 |
+
Vocabulary: <blank> + space + a-z + apostrophe
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def __init__(self):
|
| 18 |
+
self._chars = " " + "abcdefghijklmnopqrstuvwxyz" + "'"
|
| 19 |
+
self._blank = "<blank>"
|
| 20 |
+
self._char_to_idx: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
|
| 21 |
+
self._idx_to_char: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
|
| 22 |
+
# Index 0 is reserved for CTC blank
|
| 23 |
+
self._idx_to_char[0] = ""
|
| 24 |
+
|
| 25 |
+
def encode(self, text: str) -> torch.Tensor:
|
| 26 |
+
text = text.lower().strip()
|
| 27 |
+
indices = []
|
| 28 |
+
for c in text:
|
| 29 |
+
if c in self._char_to_idx:
|
| 30 |
+
indices.append(self._char_to_idx[c])
|
| 31 |
+
return torch.tensor(indices, dtype=torch.long)
|
| 32 |
+
|
| 33 |
+
def decode(self, tokens: torch.Tensor) -> str:
|
| 34 |
+
chars = []
|
| 35 |
+
prev = -1
|
| 36 |
+
for idx in tokens.tolist():
|
| 37 |
+
if idx == self.blank_id:
|
| 38 |
+
prev = idx
|
| 39 |
+
continue
|
| 40 |
+
if idx != prev:
|
| 41 |
+
chars.append(self._idx_to_char.get(idx, ""))
|
| 42 |
+
prev = idx
|
| 43 |
+
return "".join(chars)
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def vocab_size(self) -> int:
|
| 47 |
+
return len(self._chars) + 1 # +1 for blank
|
| 48 |
+
|
| 49 |
+
@property
|
| 50 |
+
def blank_id(self) -> int:
|
| 51 |
+
return 0
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class BPETokenizer(ITokenizer):
|
| 55 |
+
"""
|
| 56 |
+
BPE (Byte Pair Encoding) tokenizer for transcription.
|
| 57 |
+
|
| 58 |
+
Uses a simple word-level BPE trained on the fly from training transcripts.
|
| 59 |
+
Fewer output steps = better CTC alignment.
|
| 60 |
+
Token 0 is reserved for CTC blank.
|
| 61 |
+
"""
|
| 62 |
+
|
| 63 |
+
def __init__(self, vocab_size: int = 1000):
|
| 64 |
+
self._target_vocab_size = vocab_size
|
| 65 |
+
# Start with character-level vocab
|
| 66 |
+
self._chars = list(" abcdefghijklmnopqrstuvwxyz'")
|
| 67 |
+
# Token 0 = blank, tokens 1..N = actual symbols
|
| 68 |
+
self._token_to_id: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
|
| 69 |
+
self._id_to_token: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
|
| 70 |
+
self._id_to_token[0] = ""
|
| 71 |
+
self._trained = False
|
| 72 |
+
|
| 73 |
+
def train_from_texts(self, texts: list[str], max_merges: int | None = None) -> None:
|
| 74 |
+
"""Train BPE merges from a list of transcripts."""
|
| 75 |
+
max_merges = max_merges or (self._target_vocab_size - len(self._chars) - 1)
|
| 76 |
+
|
| 77 |
+
# Tokenize all texts into characters
|
| 78 |
+
corpus: list[list[str]] = []
|
| 79 |
+
for text in texts:
|
| 80 |
+
text = text.lower().strip()
|
| 81 |
+
chars = [c for c in text if c in self._token_to_id]
|
| 82 |
+
if chars:
|
| 83 |
+
corpus.append(chars)
|
| 84 |
+
|
| 85 |
+
# BPE merge loop
|
| 86 |
+
merges: list[tuple[str, str]] = []
|
| 87 |
+
for _ in range(max_merges):
|
| 88 |
+
# Count adjacent pairs
|
| 89 |
+
pair_counts: dict[tuple[str, str], int] = {}
|
| 90 |
+
for tokens in corpus:
|
| 91 |
+
for i in range(len(tokens) - 1):
|
| 92 |
+
pair = (tokens[i], tokens[i + 1])
|
| 93 |
+
pair_counts[pair] = pair_counts.get(pair, 0) + 1
|
| 94 |
+
|
| 95 |
+
if not pair_counts:
|
| 96 |
+
break
|
| 97 |
+
|
| 98 |
+
# Find most frequent pair
|
| 99 |
+
best_pair = max(pair_counts, key=pair_counts.get) # type: ignore
|
| 100 |
+
merged = best_pair[0] + best_pair[1]
|
| 101 |
+
merges.append(best_pair)
|
| 102 |
+
|
| 103 |
+
# Add to vocab
|
| 104 |
+
new_id = len(self._id_to_token)
|
| 105 |
+
self._token_to_id[merged] = new_id
|
| 106 |
+
self._id_to_token[new_id] = merged
|
| 107 |
+
|
| 108 |
+
# Apply merge to corpus
|
| 109 |
+
for idx, tokens in enumerate(corpus):
|
| 110 |
+
new_tokens = []
|
| 111 |
+
i = 0
|
| 112 |
+
while i < len(tokens):
|
| 113 |
+
if i < len(tokens) - 1 and (tokens[i], tokens[i + 1]) == best_pair:
|
| 114 |
+
new_tokens.append(merged)
|
| 115 |
+
i += 2
|
| 116 |
+
else:
|
| 117 |
+
new_tokens.append(tokens[i])
|
| 118 |
+
i += 1
|
| 119 |
+
corpus[idx] = new_tokens
|
| 120 |
+
|
| 121 |
+
self._merges = merges
|
| 122 |
+
self._trained = True
|
| 123 |
+
print(f" BPE trained: {len(merges)} merges, vocab size {self.vocab_size}", flush=True)
|
| 124 |
+
|
| 125 |
+
def encode(self, text: str) -> torch.Tensor:
|
| 126 |
+
text = text.lower().strip()
|
| 127 |
+
tokens = [c for c in text if c in self._token_to_id]
|
| 128 |
+
|
| 129 |
+
# Apply merges in order
|
| 130 |
+
if self._trained:
|
| 131 |
+
for a, b in self._merges:
|
| 132 |
+
merged = a + b
|
| 133 |
+
new_tokens = []
|
| 134 |
+
i = 0
|
| 135 |
+
while i < len(tokens):
|
| 136 |
+
if i < len(tokens) - 1 and tokens[i] == a and tokens[i + 1] == b:
|
| 137 |
+
new_tokens.append(merged)
|
| 138 |
+
i += 2
|
| 139 |
+
else:
|
| 140 |
+
new_tokens.append(tokens[i])
|
| 141 |
+
i += 1
|
| 142 |
+
tokens = new_tokens
|
| 143 |
+
|
| 144 |
+
indices = [self._token_to_id[t] for t in tokens if t in self._token_to_id]
|
| 145 |
+
return torch.tensor(indices, dtype=torch.long)
|
| 146 |
+
|
| 147 |
+
def decode(self, tokens: torch.Tensor) -> str:
|
| 148 |
+
pieces = []
|
| 149 |
+
prev = -1
|
| 150 |
+
for idx in tokens.tolist():
|
| 151 |
+
if idx == self.blank_id:
|
| 152 |
+
prev = idx
|
| 153 |
+
continue
|
| 154 |
+
if idx != prev:
|
| 155 |
+
pieces.append(self._id_to_token.get(idx, ""))
|
| 156 |
+
prev = idx
|
| 157 |
+
return "".join(pieces)
|
| 158 |
+
|
| 159 |
+
@property
|
| 160 |
+
def vocab_size(self) -> int:
|
| 161 |
+
return len(self._id_to_token)
|
| 162 |
+
|
| 163 |
+
@property
|
| 164 |
+
def blank_id(self) -> int:
|
| 165 |
+
return 0
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def create_tokenizer(config_type: str = "char", **kwargs) -> ITokenizer:
|
| 169 |
+
"""Factory function to create the right tokenizer."""
|
| 170 |
+
if config_type == "bpe":
|
| 171 |
+
return BPETokenizer(vocab_size=kwargs.get("vocab_size", 1000))
|
| 172 |
+
return CharTokenizer()
|
src/services/training_service.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from dataclasses import dataclass
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import torch.nn as nn
|
| 7 |
+
from torch.utils.data import DataLoader
|
| 8 |
+
|
| 9 |
+
from src.models.config import GrassmannConfig
|
| 10 |
+
from src.models.grassmann_model import GrassmannTranscriptionModel
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class TrainMetrics:
|
| 15 |
+
epoch: int
|
| 16 |
+
train_loss: float
|
| 17 |
+
val_loss: float | None = None
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class TrainingService:
|
| 21 |
+
"""Handles the training loop, loss computation, and optimization."""
|
| 22 |
+
|
| 23 |
+
def __init__(
|
| 24 |
+
self,
|
| 25 |
+
model: GrassmannTranscriptionModel,
|
| 26 |
+
config: GrassmannConfig,
|
| 27 |
+
device: torch.device | None = None,
|
| 28 |
+
):
|
| 29 |
+
self.model = model
|
| 30 |
+
self.config = config
|
| 31 |
+
if device is not None:
|
| 32 |
+
self.device = device
|
| 33 |
+
elif torch.cuda.is_available():
|
| 34 |
+
self.device = torch.device("cuda")
|
| 35 |
+
else:
|
| 36 |
+
# MPS can be unstable with some PyTorch/Python combos; prefer CPU as fallback
|
| 37 |
+
self.device = torch.device("cpu")
|
| 38 |
+
self.model.to(self.device)
|
| 39 |
+
|
| 40 |
+
self.optimizer = torch.optim.AdamW(
|
| 41 |
+
model.parameters(),
|
| 42 |
+
lr=config.learning_rate,
|
| 43 |
+
weight_decay=config.weight_decay,
|
| 44 |
+
)
|
| 45 |
+
self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
|
| 46 |
+
self.ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True)
|
| 47 |
+
self.history: list[TrainMetrics] = []
|
| 48 |
+
|
| 49 |
+
def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
|
| 50 |
+
"""Add cosine annealing LR schedule with linear warmup."""
|
| 51 |
+
from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
|
| 52 |
+
warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
|
| 53 |
+
cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
|
| 54 |
+
self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
|
| 55 |
+
|
| 56 |
+
def train_epoch(self, train_loader: DataLoader) -> float:
|
| 57 |
+
self.model.train()
|
| 58 |
+
total_loss = 0.0
|
| 59 |
+
n_batches = 0
|
| 60 |
+
total_batches = len(train_loader)
|
| 61 |
+
|
| 62 |
+
for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
|
| 63 |
+
mels = mels.to(self.device)
|
| 64 |
+
tokens = tokens.to(self.device)
|
| 65 |
+
mel_lengths = mel_lengths.to(self.device)
|
| 66 |
+
token_lengths = token_lengths.to(self.device)
|
| 67 |
+
|
| 68 |
+
self.optimizer.zero_grad()
|
| 69 |
+
|
| 70 |
+
log_probs = self.model(mels) # (T', B, vocab)
|
| 71 |
+
output_lengths = self.model.get_output_lengths(mel_lengths)
|
| 72 |
+
|
| 73 |
+
loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
|
| 74 |
+
|
| 75 |
+
if torch.isfinite(loss):
|
| 76 |
+
loss.backward()
|
| 77 |
+
torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
|
| 78 |
+
self.optimizer.step()
|
| 79 |
+
total_loss += loss.item()
|
| 80 |
+
n_batches += 1
|
| 81 |
+
|
| 82 |
+
if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
|
| 83 |
+
avg = total_loss / max(n_batches, 1)
|
| 84 |
+
print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
|
| 85 |
+
|
| 86 |
+
return total_loss / max(n_batches, 1)
|
| 87 |
+
|
| 88 |
+
@torch.no_grad()
|
| 89 |
+
def validate(self, val_loader: DataLoader) -> float:
|
| 90 |
+
self.model.eval()
|
| 91 |
+
total_loss = 0.0
|
| 92 |
+
n_batches = 0
|
| 93 |
+
|
| 94 |
+
for mels, tokens, mel_lengths, token_lengths in val_loader:
|
| 95 |
+
mels = mels.to(self.device)
|
| 96 |
+
tokens = tokens.to(self.device)
|
| 97 |
+
mel_lengths = mel_lengths.to(self.device)
|
| 98 |
+
token_lengths = token_lengths.to(self.device)
|
| 99 |
+
|
| 100 |
+
log_probs = self.model(mels)
|
| 101 |
+
output_lengths = self.model.get_output_lengths(mel_lengths)
|
| 102 |
+
|
| 103 |
+
loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
|
| 104 |
+
if torch.isfinite(loss):
|
| 105 |
+
total_loss += loss.item()
|
| 106 |
+
n_batches += 1
|
| 107 |
+
|
| 108 |
+
return total_loss / max(n_batches, 1)
|
| 109 |
+
|
| 110 |
+
def train_loop(
|
| 111 |
+
self,
|
| 112 |
+
train_loader: DataLoader,
|
| 113 |
+
val_loader: DataLoader | None,
|
| 114 |
+
epochs: int,
|
| 115 |
+
callback: object | None = None,
|
| 116 |
+
) -> list[TrainMetrics]:
|
| 117 |
+
for epoch in range(1, epochs + 1):
|
| 118 |
+
train_loss = self.train_epoch(train_loader)
|
| 119 |
+
val_loss = self.validate(val_loader) if val_loader else None
|
| 120 |
+
|
| 121 |
+
metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
|
| 122 |
+
self.history.append(metrics)
|
| 123 |
+
|
| 124 |
+
if callback and hasattr(callback, "on_epoch_end"):
|
| 125 |
+
callback.on_epoch_end(metrics)
|
| 126 |
+
|
| 127 |
+
if self.scheduler is not None:
|
| 128 |
+
self.scheduler.step()
|
| 129 |
+
|
| 130 |
+
return self.history
|
| 131 |
+
|
| 132 |
+
@torch.no_grad()
|
| 133 |
+
def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
|
| 134 |
+
"""Greedy CTC decoding for a single mel spectrogram."""
|
| 135 |
+
self.model.eval()
|
| 136 |
+
if mel.dim() == 2:
|
| 137 |
+
mel = mel.unsqueeze(0) # add batch dim
|
| 138 |
+
mel = mel.to(self.device)
|
| 139 |
+
log_probs = self.model(mel) # (T', 1, vocab)
|
| 140 |
+
pred = log_probs.argmax(dim=-1).squeeze(1) # (T',)
|
| 141 |
+
return pred
|
src/services/upload_service.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
UploadThing upload service — uploads files directly via signed URLs.
|
| 3 |
+
|
| 4 |
+
Generates file keys locally (Sqids + djb2), signs the upload URL with HMAC-SHA256,
|
| 5 |
+
and PUTs directly to the regional ingest endpoint. No server-side API call needed.
|
| 6 |
+
|
| 7 |
+
Requires UPLOADTHING_TOKEN env var (base64-encoded JSON with apiKey, appId, regions).
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import base64
|
| 11 |
+
import hashlib
|
| 12 |
+
import hmac
|
| 13 |
+
import json
|
| 14 |
+
import math
|
| 15 |
+
import os
|
| 16 |
+
import time
|
| 17 |
+
import urllib.error
|
| 18 |
+
import urllib.request
|
| 19 |
+
import uuid
|
| 20 |
+
|
| 21 |
+
from sqids import Sqids
|
| 22 |
+
from sqids.constants import DEFAULT_ALPHABET
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _djb2(s: str) -> int:
|
| 26 |
+
"""DJB2 hash matching UploadThing's JS implementation."""
|
| 27 |
+
h = 5381
|
| 28 |
+
for ch in reversed(s):
|
| 29 |
+
h = (h * 33) ^ ord(ch)
|
| 30 |
+
h &= 0xFFFFFFFF
|
| 31 |
+
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)
|
| 32 |
+
if h >= 0x80000000:
|
| 33 |
+
h -= 0x100000000
|
| 34 |
+
return h
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _shuffle(input_str: str, seed: str) -> str:
|
| 38 |
+
"""Shuffle alphabet deterministically using seed (matches UploadThing JS)."""
|
| 39 |
+
chars = list(input_str)
|
| 40 |
+
seed_num = _djb2(seed)
|
| 41 |
+
for i in range(len(chars)):
|
| 42 |
+
j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars)))
|
| 43 |
+
chars[i], chars[j] = chars[j], chars[i]
|
| 44 |
+
return "".join(chars)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _generate_file_key(app_id: str) -> str:
|
| 48 |
+
"""Generate a unique UploadThing file key using Sqids (matches JS SDK)."""
|
| 49 |
+
alphabet = _shuffle(DEFAULT_ALPHABET, app_id)
|
| 50 |
+
sqids = Sqids(alphabet=alphabet, min_length=12)
|
| 51 |
+
encoded_app_id = sqids.encode([abs(_djb2(app_id))])
|
| 52 |
+
file_seed = base64.urlsafe_b64encode(uuid.uuid4().bytes).decode().rstrip("=")
|
| 53 |
+
return encoded_app_id + file_seed
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _pct_encode(s: str) -> str:
|
| 57 |
+
"""Percent-encode like JS encodeURIComponent."""
|
| 58 |
+
return urllib.request.quote(s, safe="")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _generate_signed_url(
|
| 62 |
+
base_url: str, api_key: str, app_id: str,
|
| 63 |
+
file_name: str, file_size: int,
|
| 64 |
+
file_type: str = "application/octet-stream",
|
| 65 |
+
) -> str:
|
| 66 |
+
"""Generate HMAC-signed upload URL."""
|
| 67 |
+
expires = int(time.time() * 1000) + 3600 * 1000 # 1 hour
|
| 68 |
+
|
| 69 |
+
params = (
|
| 70 |
+
f"?expires={expires}"
|
| 71 |
+
f"&x-ut-identifier={_pct_encode(app_id)}"
|
| 72 |
+
f"&x-ut-file-name={_pct_encode(file_name)}"
|
| 73 |
+
f"&x-ut-file-size={file_size}"
|
| 74 |
+
f"&x-ut-file-type={_pct_encode(file_type)}"
|
| 75 |
+
f"&x-ut-content-disposition=inline"
|
| 76 |
+
f"&x-ut-acl=public-read"
|
| 77 |
+
)
|
| 78 |
+
url = base_url + params
|
| 79 |
+
sig = hmac.new(api_key.encode(), url.encode(), hashlib.sha256).hexdigest()
|
| 80 |
+
return f"{url}&signature=hmac-sha256={sig}"
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _decode_token() -> tuple[str, str, str]:
|
| 84 |
+
"""Decode UPLOADTHING_TOKEN to get (api_key, app_id, region)."""
|
| 85 |
+
token = os.environ.get("UPLOADTHING_TOKEN")
|
| 86 |
+
if not token:
|
| 87 |
+
raise RuntimeError("UPLOADTHING_TOKEN env var not set")
|
| 88 |
+
decoded = json.loads(base64.b64decode(token))
|
| 89 |
+
return decoded["apiKey"], decoded["appId"], decoded.get("regions", ["sea1"])[0]
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _build_multipart(file_name: str, file_data: bytes, content_type: str) -> tuple[bytes, str]:
|
| 93 |
+
"""Build multipart/form-data body manually (no requests dependency)."""
|
| 94 |
+
boundary = uuid.uuid4().hex
|
| 95 |
+
body = (
|
| 96 |
+
f"--{boundary}\r\n"
|
| 97 |
+
f'Content-Disposition: form-data; name="file"; filename="{file_name}"\r\n'
|
| 98 |
+
f"Content-Type: {content_type}\r\n"
|
| 99 |
+
f"\r\n"
|
| 100 |
+
).encode() + file_data + f"\r\n--{boundary}--\r\n".encode()
|
| 101 |
+
return body, f"multipart/form-data; boundary={boundary}"
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def upload_file(file_path: str, custom_id: str | None = None) -> str | None:
|
| 105 |
+
"""
|
| 106 |
+
Upload a single file to UploadThing.
|
| 107 |
+
|
| 108 |
+
Returns the ufsUrl of the uploaded file, or None on failure.
|
| 109 |
+
"""
|
| 110 |
+
if not os.path.isfile(file_path):
|
| 111 |
+
print(f" [upload] File not found: {file_path}", flush=True)
|
| 112 |
+
return None
|
| 113 |
+
|
| 114 |
+
api_key, app_id, region = _decode_token()
|
| 115 |
+
file_name = os.path.basename(file_path)
|
| 116 |
+
file_size = os.path.getsize(file_path)
|
| 117 |
+
|
| 118 |
+
# Generate file key and signed URL
|
| 119 |
+
file_key = _generate_file_key(app_id)
|
| 120 |
+
ingest_base = f"https://{region}.ingest.uploadthing.com/{file_key}"
|
| 121 |
+
signed_url = _generate_signed_url(
|
| 122 |
+
ingest_base, api_key, app_id, file_name, file_size,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
print(f" [upload] Uploading {file_name} ({file_size / 1e6:.1f} MB)...", flush=True)
|
| 126 |
+
|
| 127 |
+
with open(file_path, "rb") as f:
|
| 128 |
+
file_data = f.read()
|
| 129 |
+
|
| 130 |
+
# PUT with multipart form-data (matching the working Python/C# implementations)
|
| 131 |
+
body, content_type = _build_multipart(file_name, file_data, "application/octet-stream")
|
| 132 |
+
|
| 133 |
+
put_req = urllib.request.Request(
|
| 134 |
+
signed_url,
|
| 135 |
+
data=body,
|
| 136 |
+
headers={
|
| 137 |
+
"Content-Type": content_type,
|
| 138 |
+
"Range": "bytes=0-",
|
| 139 |
+
},
|
| 140 |
+
method="PUT",
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
try:
|
| 144 |
+
with urllib.request.urlopen(put_req, timeout=300) as resp:
|
| 145 |
+
resp_body = resp.read().decode()
|
| 146 |
+
resp_json = json.loads(resp_body)
|
| 147 |
+
ufs_url = resp_json.get("ufsUrl") or resp_json.get("url", "")
|
| 148 |
+
print(f" [upload] Success: {ufs_url}", flush=True)
|
| 149 |
+
return ufs_url
|
| 150 |
+
except urllib.error.HTTPError as e:
|
| 151 |
+
body_text = e.read().decode() if e.fp else ""
|
| 152 |
+
print(f" [upload] PUT failed: {e.code} {body_text[:500]}", flush=True)
|
| 153 |
+
return None
|
| 154 |
+
except Exception as e:
|
| 155 |
+
print(f" [upload] PUT error: {e}", flush=True)
|
| 156 |
+
return None
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def upload_directory(dir_path: str, prefix: str = "") -> list[str]:
|
| 160 |
+
"""Upload all files in a directory. Returns list of uploaded URLs."""
|
| 161 |
+
urls = []
|
| 162 |
+
if not os.path.isdir(dir_path):
|
| 163 |
+
print(f" [upload] Directory not found: {dir_path}", flush=True)
|
| 164 |
+
return urls
|
| 165 |
+
|
| 166 |
+
for fname in sorted(os.listdir(dir_path)):
|
| 167 |
+
fpath = os.path.join(dir_path, fname)
|
| 168 |
+
if os.path.isfile(fpath):
|
| 169 |
+
url = upload_file(fpath, custom_id=f"{prefix}/{fname}" if prefix else fname)
|
| 170 |
+
if url:
|
| 171 |
+
urls.append(url)
|
| 172 |
+
|
| 173 |
+
return urls
|
src/views/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
from .console_view import ConsoleView
|
src/views/console_view.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Console view: prints training progress and results."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import TYPE_CHECKING
|
| 6 |
+
|
| 7 |
+
if TYPE_CHECKING:
|
| 8 |
+
from src.models.config import GrassmannConfig
|
| 9 |
+
from src.services.training_service import TrainMetrics
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ConsoleView:
|
| 13 |
+
"""Simple console-based display for training progress."""
|
| 14 |
+
|
| 15 |
+
def show_model_info(self, config: GrassmannConfig, param_count: int) -> None:
|
| 16 |
+
mode = "Seq2Seq (Encoder-Decoder)" if config.use_seq2seq else "CTC (Encoder-only)"
|
| 17 |
+
print("\n" + "=" * 60)
|
| 18 |
+
print(f" Grassmann Transcription Model — {mode}")
|
| 19 |
+
print("=" * 60)
|
| 20 |
+
print(f" Encoder layers: {config.n_layers}")
|
| 21 |
+
if config.use_seq2seq:
|
| 22 |
+
print(f" Decoder layers: {config.n_decoder_layers}")
|
| 23 |
+
print(f" Cross window: {config.cross_window}")
|
| 24 |
+
print(f" d_model: {config.d_model}")
|
| 25 |
+
print(f" r_reduced: {config.r_reduced}")
|
| 26 |
+
print(f" Plücker dim: {config.plucker_dim}")
|
| 27 |
+
print(f" Window offsets: {config.window_offsets}")
|
| 28 |
+
print(f" Vocab size: {config.vocab_size}")
|
| 29 |
+
print(f" Parameters: {param_count:,}")
|
| 30 |
+
print(f" Attention: NONE (fully Grassmann)")
|
| 31 |
+
print("=" * 60 + "\n")
|
| 32 |
+
|
| 33 |
+
def on_epoch_end(self, metrics: TrainMetrics) -> None:
|
| 34 |
+
val_str = f" val_loss: {metrics.val_loss:.4f}" if metrics.val_loss is not None else ""
|
| 35 |
+
print(f" Epoch {metrics.epoch:3d} | train_loss: {metrics.train_loss:.4f}{val_str}")
|
| 36 |
+
|
| 37 |
+
def show_training_complete(self, history: list[TrainMetrics]) -> None:
|
| 38 |
+
if history:
|
| 39 |
+
best = min(history, key=lambda m: m.val_loss if m.val_loss is not None else m.train_loss)
|
| 40 |
+
print(f"\n Best epoch: {best.epoch} | loss: {best.val_loss or best.train_loss:.4f}")
|
| 41 |
+
print(" Training complete.\n")
|
| 42 |
+
|
| 43 |
+
def show_test_results(self, results: dict) -> None:
|
| 44 |
+
print("\n Test Results:")
|
| 45 |
+
for k, v in results.items():
|
| 46 |
+
print(f" {k}: {v:.4f}")
|
| 47 |
+
print()
|
| 48 |
+
|
| 49 |
+
def show_message(self, msg: str) -> None:
|
| 50 |
+
print(f" {msg}")
|