Nicholas Finch commited on
Commit
d7013e6
·
0 Parent(s):

fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic

Browse files
Dockerfile ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.11-slim
2
+
3
+ WORKDIR /app
4
+
5
+ # System deps for audio decoding
6
+ RUN apt-get update && apt-get install -y --no-install-recommends \
7
+ libsndfile1 ffmpeg git \
8
+ && rm -rf /var/lib/apt/lists/*
9
+
10
+ # Python deps — CPU-only torch (smaller image, faster build)
11
+ RUN pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cpu
12
+ COPY requirements.txt .
13
+ RUN pip install --no-cache-dir -r requirements.txt
14
+ RUN pip install --no-cache-dir 'datasets<4' librosa soundfile huggingface_hub
15
+
16
+ # Copy project
17
+ COPY src/ src/
18
+ COPY app.py .
19
+ COPY recipes.yaml .
20
+
21
+ # Cache 1750h recipe and upload to HF Dataset repo
22
+ # HF_TOKEN must be set as a Space secret for upload
23
+ CMD ["python", "-u", "app.py", "cache", \
24
+ "--recipe", "1750h", \
25
+ "--cache-dir", "mel_cache", \
26
+ "--repo", "tisaconundrum/grassmann-mel-cache"]
README.md ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Grassmann Cache
3
+ emoji: 📦
4
+ colorFrom: blue
5
+ colorTo: green
6
+ sdk: docker
7
+ pinned: false
8
+ ---
9
+
10
+ Caches 1750h of mel spectrograms for Grassmann ASR training.
app.py ADDED
@@ -0,0 +1,518 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Grassmann Transcription — Unified CLI
4
+
5
+ One entry point for training, caching, evaluation, export, and inference.
6
+ Based on "Attention Is Not What You Need" (arXiv:2512.19428).
7
+
8
+ Usage:
9
+ python app.py train --from-cache tisaconundrum/grassmann-mel-cache
10
+ python app.py train --recipe 1750h
11
+ python app.py cache --recipe 1750h --repo tisaconundrum/grassmann-mel-cache
12
+ python app.py export --model checkpoints/bundle --output onnx_export/
13
+ python app.py eval --model eval_model/best_model.pt --config eval_model/config.json
14
+ python app.py transcribe --model checkpoints/bundle --audio speech.wav
15
+ python app.py demo
16
+ """
17
+
18
+ import argparse
19
+ import os
20
+ import sys
21
+ import time
22
+ import threading
23
+
24
+ import torch
25
+
26
+ os.environ["PYTHONUNBUFFERED"] = "1"
27
+ sys.stdout.reconfigure(line_buffering=True)
28
+
29
+
30
+ # ── Shared utilities ─────────────────────────────────────────────────────
31
+
32
+ def _start_health_server(port: int = 7860) -> None:
33
+ from http.server import HTTPServer, BaseHTTPRequestHandler
34
+
35
+ class Handler(BaseHTTPRequestHandler):
36
+ def do_GET(self):
37
+ self.send_response(200)
38
+ self.send_header("Content-Type", "text/plain")
39
+ self.end_headers()
40
+ self.wfile.write(b"Running...")
41
+
42
+ def log_message(self, format, *args):
43
+ pass
44
+
45
+ server = HTTPServer(("0.0.0.0", port), Handler)
46
+ threading.Thread(target=server.serve_forever, daemon=True).start()
47
+ print(f" Health server on port {port}", flush=True)
48
+
49
+
50
+ def _log_memory(label: str) -> None:
51
+ try:
52
+ import resource
53
+ rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
54
+ rss_mb = rss_kb / (1024 * 1024 if sys.platform == "darwin" else 1024)
55
+ print(f" [mem] {label}: RSS={rss_mb:.0f}MB", end="", flush=True)
56
+ except Exception:
57
+ print(f" [mem] {label}: RSS=unknown", end="", flush=True)
58
+ if torch.cuda.is_available():
59
+ alloc = torch.cuda.memory_allocated() / 1e6
60
+ reserved = torch.cuda.memory_reserved() / 1e6
61
+ print(f" GPU_alloc={alloc:.0f}MB GPU_reserved={reserved:.0f}MB", flush=True)
62
+ else:
63
+ print(flush=True)
64
+
65
+
66
+ def _add_model_args(p: argparse.ArgumentParser) -> None:
67
+ p.add_argument("--layers", type=int, default=6)
68
+ p.add_argument("--d-model", type=int, default=256)
69
+ p.add_argument("--r-reduced", type=int, default=32)
70
+ p.add_argument("--lr", type=float, default=3e-4)
71
+ p.add_argument("--decoder-layers", type=int, default=4)
72
+ p.add_argument("--cross-window", type=int, default=8)
73
+ p.add_argument("--ctc", action="store_true", help="CTC-only mode")
74
+ p.add_argument("--conformer", action="store_true", help="Conformer blocks")
75
+ p.add_argument("--two-sided", action="store_true", help="Bidirectional mixing")
76
+ p.add_argument("--conv-kernel", type=int, default=31)
77
+ p.add_argument("--ablation", action="store_true", help="Symmetric-product ablation (no Grassmann geometry)")
78
+
79
+
80
+ def _auto_pause_space() -> None:
81
+ try:
82
+ from huggingface_hub import HfApi
83
+ space_id = os.environ.get("SPACE_ID")
84
+ if space_id:
85
+ HfApi().pause_space(space_id)
86
+ print(f" Space {space_id} paused.", flush=True)
87
+ except Exception as e:
88
+ print(f" Note: could not auto-pause: {e}", flush=True)
89
+
90
+
91
+ # ── train ────────────────────────────────────────────────────────────────
92
+
93
+ def cmd_train(args: argparse.Namespace) -> None:
94
+ import gc
95
+ from src.models.config import GrassmannConfig
96
+ from src.controllers.model_controller import ModelController
97
+ from src.services.tokenizer_service import CharTokenizer
98
+ from src.repositories.dataset_repository import DatasetRepository
99
+ from src.repositories.cache_repository import CacheRepository
100
+ from src.services.dataset_service import DatasetService
101
+
102
+ _start_health_server()
103
+
104
+ config = GrassmannConfig(
105
+ epochs=args.epochs, batch_size=args.batch_size,
106
+ n_layers=args.layers, n_decoder_layers=args.decoder_layers,
107
+ cross_window=args.cross_window, d_model=args.d_model,
108
+ r_reduced=args.r_reduced, learning_rate=args.lr,
109
+ use_seq2seq=not args.ctc, use_conformer=args.conformer,
110
+ two_sided=args.two_sided, conv_kernel_size=args.conv_kernel,
111
+ use_ablation=args.ablation,
112
+ )
113
+
114
+ print("\n" + "=" * 60)
115
+ print(" Grassmann Transcription — Training")
116
+ print("=" * 60)
117
+ if torch.cuda.is_available():
118
+ gpu = torch.cuda.get_device_name(0)
119
+ mem = torch.cuda.get_device_properties(0).total_memory / 1e9
120
+ print(f" GPU: {gpu} ({mem:.1f} GB)")
121
+ else:
122
+ print(" GPU: None (CPU)")
123
+ mode = "CTC" if args.ctc else "Seq2Seq"
124
+ block = "Conformer" if args.conformer else "Plain"
125
+ mix = "two-sided" if args.two_sided else "causal"
126
+ mixer = "Ablation (symmetric)" if args.ablation else "Grassmann (Plücker)"
127
+ print(f" Mode: {mode} | Blocks: {block} ({mix})")
128
+ print(f" Mixer: {mixer}")
129
+ print(f" Encoder: {args.layers}L / d={args.d_model} / r={args.r_reduced}")
130
+ print("=" * 60 + "\n")
131
+
132
+ tokenizer = CharTokenizer()
133
+ cache_dir = os.path.join(args.save_dir, "mel_cache")
134
+ repo = DatasetRepository(config, tokenizer)
135
+ cache = CacheRepository(cache_dir)
136
+ dataset = DatasetService(repo, cache)
137
+
138
+ if args.from_cache:
139
+ dataset.load_from_hub_cache(args.from_cache).clean().preprocess().split(
140
+ max_samples=args.max_samples)
141
+ elif args.recipe:
142
+ dataset.load_recipe(args.recipe,
143
+ max_samples_per_source=args.max_per_source).clean().preprocess().split()
144
+ else:
145
+ dataset.load_dataset(args.dataset, split=args.split,
146
+ max_samples=args.max_samples).clean().preprocess().split()
147
+
148
+ gc.collect()
149
+ if torch.cuda.is_available():
150
+ torch.cuda.empty_cache()
151
+ _log_memory("After dataset load")
152
+
153
+ model_ctrl = ModelController(config, tokenizer)
154
+ model_ctrl.build()
155
+ assert model_ctrl.trainer is not None
156
+ model_ctrl.trainer.set_cosine_schedule(args.epochs, warmup_epochs=min(3, args.epochs // 5))
157
+
158
+ train_loader = dataset.get_train_loader(config.batch_size)
159
+ val_loader = dataset.get_val_loader(config.batch_size)
160
+
161
+ os.makedirs(args.save_dir, exist_ok=True)
162
+ best_val_loss = float("inf")
163
+ start_time = time.time()
164
+ _log_memory("Before training")
165
+
166
+ for epoch in range(1, args.epochs + 1):
167
+ epoch_start = time.time()
168
+ train_loss = model_ctrl.trainer.train_epoch(train_loader)
169
+ val_loss = model_ctrl.trainer.validate(val_loader)
170
+
171
+ gc.collect()
172
+ if torch.cuda.is_available():
173
+ torch.cuda.empty_cache()
174
+
175
+ if model_ctrl.trainer.scheduler is not None:
176
+ model_ctrl.trainer.scheduler.step()
177
+
178
+ lr = model_ctrl.trainer.optimizer.param_groups[0]["lr"]
179
+ elapsed = time.time() - epoch_start
180
+ total_elapsed = time.time() - start_time
181
+ mem_str = ""
182
+ if torch.cuda.is_available():
183
+ alloc = torch.cuda.memory_allocated() / 1e9
184
+ reserved = torch.cuda.memory_reserved() / 1e9
185
+ mem_str = f" GPU: {alloc:.1f}/{reserved:.1f}GB"
186
+
187
+ print(f" Epoch {epoch:3d}/{args.epochs} | "
188
+ f"train: {train_loss:.4f} val: {val_loss:.4f} "
189
+ f"lr: {lr:.2e} ({elapsed:.1f}s, total: {total_elapsed/60:.1f}min)"
190
+ f"{mem_str}", flush=True)
191
+
192
+ if val_loss < best_val_loss:
193
+ best_val_loss = val_loss
194
+ assert model_ctrl.nn_model is not None
195
+ torch.save(model_ctrl.nn_model.state_dict(),
196
+ os.path.join(args.save_dir, "best_model.pt"))
197
+ print(f" -> Saved best model (val_loss={val_loss:.4f})", flush=True)
198
+
199
+ if epoch % 10 == 0:
200
+ assert model_ctrl.nn_model is not None
201
+ torch.save(model_ctrl.nn_model.state_dict(),
202
+ os.path.join(args.save_dir, f"epoch_{epoch}.pt"))
203
+
204
+ if epoch % args.decode_every == 0 or epoch == 1:
205
+ _decode_samples(model_ctrl, dataset)
206
+
207
+ assert model_ctrl.nn_model is not None
208
+ torch.save(model_ctrl.nn_model.state_dict(), os.path.join(args.save_dir, "final_model.pt"))
209
+
210
+ test_loader = dataset.get_test_loader(config.batch_size)
211
+ test_loss = model_ctrl.trainer.validate(test_loader)
212
+ total_time = time.time() - start_time
213
+
214
+ print(f"\n{'=' * 60}")
215
+ print(f" Training Complete!")
216
+ print(f" Best val loss: {best_val_loss:.4f}")
217
+ print(f" Test loss: {test_loss:.4f}")
218
+ print(f" Total time: {total_time/60:.1f} minutes")
219
+ print(f"{'=' * 60}")
220
+
221
+ _decode_samples(model_ctrl, dataset, n=5)
222
+ _post_training(model_ctrl, config, args.save_dir, run_name=args.run_name)
223
+
224
+
225
+ def _decode_samples(model_ctrl, dataset, n: int = 3) -> None:
226
+ assert model_ctrl.trainer is not None and dataset._val_ds is not None
227
+ print("\n Sample transcriptions:", flush=True)
228
+ for i in range(min(n, len(dataset._val_ds))):
229
+ mel, tokens = dataset._val_ds[i]
230
+ pred_text = model_ctrl.predict(mel)
231
+ gt_text = model_ctrl.tokenizer.decode(tokens)
232
+ print(f" [{i}] GT: '{gt_text}'", flush=True)
233
+ print(f" [{i}] PRED: '{pred_text}'")
234
+ print()
235
+
236
+
237
+ def _post_training(model_ctrl, config, save_dir: str, run_name: str | None = None) -> None:
238
+ from src.services.export_service import export_seq2seq, export_ctc
239
+
240
+ def _prefixed(name: str) -> str:
241
+ """Prefix a filename with run_name so uploads are distinguishable."""
242
+ if not run_name:
243
+ return name
244
+ base, ext = os.path.splitext(name)
245
+ return f"{run_name}-{base}{ext}"
246
+
247
+ try:
248
+ from src.services.model_hub_service import save_model
249
+ assert model_ctrl.nn_model is not None
250
+ save_model(model_ctrl.nn_model, config, os.path.join(save_dir, "bundle"))
251
+ print(" Model bundle saved.", flush=True)
252
+ except Exception as e:
253
+ print(f" Warning: bundle save failed: {e}", flush=True)
254
+
255
+ try:
256
+ onnx_dir = os.path.join(save_dir, "onnx")
257
+ if config.use_seq2seq:
258
+ export_seq2seq(model_ctrl.nn_model, config, onnx_dir)
259
+ else:
260
+ export_ctc(model_ctrl.nn_model, config, onnx_dir)
261
+ print(f" ONNX exported to {onnx_dir}/", flush=True)
262
+ except Exception as e:
263
+ print(f" Warning: ONNX export failed: {e}", flush=True)
264
+
265
+ # Rename artifacts with run_name prefix before upload
266
+ if run_name:
267
+ import shutil
268
+ # Rename best_model.pt → ablation-smol-best_model.pt etc.
269
+ for name in ["best_model.pt", "final_model.pt"]:
270
+ src = os.path.join(save_dir, name)
271
+ dst = os.path.join(save_dir, _prefixed(name))
272
+ if os.path.isfile(src):
273
+ shutil.move(src, dst)
274
+ # Rename ONNX files (model.onnx → ablation-smol-model.onnx etc.)
275
+ onnx_dir = os.path.join(save_dir, "onnx")
276
+ if os.path.isdir(onnx_dir):
277
+ for fname in list(os.listdir(onnx_dir)):
278
+ src = os.path.join(onnx_dir, fname)
279
+ dst = os.path.join(onnx_dir, _prefixed(fname))
280
+ if os.path.isfile(src):
281
+ shutil.move(src, dst)
282
+ # Rename bundle config files
283
+ bundle_dir = os.path.join(save_dir, "bundle")
284
+ if os.path.isdir(bundle_dir):
285
+ for fname in list(os.listdir(bundle_dir)):
286
+ src = os.path.join(bundle_dir, fname)
287
+ dst = os.path.join(bundle_dir, _prefixed(fname))
288
+ if os.path.isfile(src):
289
+ shutil.move(src, dst)
290
+
291
+ try:
292
+ from src.services.upload_service import upload_file, upload_directory
293
+ bundle_dir = os.path.join(save_dir, "bundle")
294
+ onnx_dir = os.path.join(save_dir, "onnx")
295
+ if os.path.isdir(bundle_dir):
296
+ for fname in sorted(os.listdir(bundle_dir)):
297
+ fpath = os.path.join(bundle_dir, fname)
298
+ if os.path.isfile(fpath):
299
+ upload_file(fpath)
300
+ if os.path.isdir(onnx_dir):
301
+ urls = upload_directory(onnx_dir)
302
+ print(f" Uploaded {len(urls)} ONNX files", flush=True)
303
+ best_name = _prefixed("best_model.pt")
304
+ best_path = os.path.join(save_dir, best_name)
305
+ if os.path.isfile(best_path):
306
+ upload_file(best_path)
307
+ except Exception as e:
308
+ print(f" Warning: upload failed: {e}", flush=True)
309
+
310
+ _auto_pause_space()
311
+
312
+
313
+ # ── cache ────────────────────────────────────────────────────────────────
314
+
315
+ def cmd_cache(args: argparse.Namespace) -> None:
316
+ from src.models.config import GrassmannConfig
317
+ from src.services.tokenizer_service import CharTokenizer
318
+ from src.repositories.dataset_repository import DatasetRepository
319
+ from src.repositories.cache_repository import CacheRepository
320
+ from src.services.dataset_service import DatasetService
321
+
322
+ _start_health_server()
323
+
324
+ print("\n" + "=" * 60)
325
+ print(" Grassmann — Dataset Caching (CPU)")
326
+ print("=" * 60)
327
+ print(f" Recipe: {args.recipe}")
328
+ print(f" Upload to: {args.repo}")
329
+ print("=" * 60 + "\n")
330
+
331
+ config = GrassmannConfig()
332
+ tokenizer = CharTokenizer()
333
+ repo = DatasetRepository(config, tokenizer)
334
+ cache = CacheRepository(args.cache_dir)
335
+ dataset = DatasetService(repo, cache)
336
+
337
+ start = time.time()
338
+ dataset.load_recipe(args.recipe,
339
+ max_samples_per_source=args.max_per_source,
340
+ upload_repo=args.repo,
341
+ workers=args.workers)
342
+ print(f"\n Caching complete in {(time.time() - start)/60:.1f} minutes")
343
+
344
+ _auto_pause_space()
345
+ print(f"\n Done! Now train with: python app.py train --from-cache {args.repo}")
346
+
347
+
348
+ # ── export ───────────────────────────────────────────────────────────────
349
+
350
+ def cmd_export(args: argparse.Namespace) -> None:
351
+ import json
352
+ from dataclasses import asdict
353
+ from pathlib import Path
354
+ from src.services.model_hub_service import load_model
355
+ from src.services.export_service import export_seq2seq, export_ctc, verify_onnx
356
+
357
+ print(f"\n Loading model from {args.model}...")
358
+ model, config, tokenizer = load_model(args.model, device="cpu")
359
+ print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
360
+ print(f" Mode: {'seq2seq' if config.use_seq2seq else 'CTC'}")
361
+
362
+ if config.use_seq2seq:
363
+ export_seq2seq(model, config, args.output, args.opset)
364
+ else:
365
+ export_ctc(model, config, args.output, args.opset)
366
+
367
+ out = Path(args.output)
368
+ with open(out / "config.json", "w") as f:
369
+ json.dump(asdict(config), f, indent=2)
370
+ with open(out / "tokenizer_info.json", "w") as f:
371
+ json.dump({"type": "char", "vocab_size": tokenizer.vocab_size}, f, indent=2)
372
+
373
+ print(f"\n Export complete!")
374
+ if args.verify:
375
+ verify_onnx(args.output, config, config.use_seq2seq)
376
+
377
+
378
+ # ── eval ─────────────────────────────────────────────────────────────────
379
+
380
+ def cmd_eval(args: argparse.Namespace) -> None:
381
+ import json
382
+ from src.models.config import GrassmannConfig
383
+ from src.models.grassmann_model import GrassmannTranscriptionModel
384
+ from src.services.tokenizer_service import CharTokenizer
385
+ from src.services.eval_service import evaluate_wer
386
+
387
+ print(" Loading model...", flush=True)
388
+ with open(args.config) as f:
389
+ config_dict = json.load(f)
390
+ config = GrassmannConfig(**{
391
+ k: v for k, v in config_dict.items()
392
+ if k in GrassmannConfig.__dataclass_fields__
393
+ })
394
+ tokenizer = CharTokenizer()
395
+ config.vocab_size = tokenizer.vocab_size
396
+
397
+ model = GrassmannTranscriptionModel(config)
398
+ model.load_state_dict(torch.load(args.model, map_location="cpu", weights_only=True))
399
+ model.eval()
400
+ print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
401
+
402
+ evaluate_wer(model, config, tokenizer, split=args.split,
403
+ max_samples=args.max_samples, show_errors=args.show_errors)
404
+
405
+
406
+ # ── transcribe ───────────────────────────────────────────────────────────
407
+
408
+ def cmd_transcribe(args: argparse.Namespace) -> None:
409
+ from src.models.config import GrassmannConfig
410
+ from src.controllers.pipeline_controller import PipelineController
411
+
412
+ config = GrassmannConfig()
413
+ pipeline = PipelineController(config)
414
+ pipeline.load_model(args.model)
415
+ text = pipeline.transcribe(args.audio)
416
+ print(f"\n {text}\n")
417
+
418
+
419
+ # ── demo ─────────────────────────────────────────────────────────────────
420
+
421
+ def cmd_demo() -> None:
422
+ from src.models.config import GrassmannConfig
423
+ from src.controllers.model_controller import ModelController
424
+ from src.services.tokenizer_service import CharTokenizer
425
+
426
+ print("\n Running demo with synthetic data...\n")
427
+ config = GrassmannConfig(n_layers=2, d_model=128, r_reduced=16, d_ff=512,
428
+ n_decoder_layers=2, epochs=3, batch_size=4)
429
+ tokenizer = CharTokenizer()
430
+ config.vocab_size = tokenizer.vocab_size
431
+
432
+ model = ModelController(config, tokenizer)
433
+ model.build()
434
+ assert model.trainer is not None
435
+ device = model.trainer.device
436
+ mels = torch.randn(4, config.n_mels, 200, device=device)
437
+
438
+ assert model.nn_model is not None
439
+ if config.use_seq2seq:
440
+ tokens = tokenizer.encode("hello world").unsqueeze(0).expand(4, -1)
441
+ logits = model.nn_model(mels, tokens)
442
+ print(f" Forward pass OK — output: {logits.shape}")
443
+ else:
444
+ log_probs = model.nn_model(mels)
445
+ print(f" Forward pass OK — output: {log_probs.shape}")
446
+
447
+ pred_tokens = model.trainer.greedy_decode(mels[0:1])
448
+ decoded = tokenizer.decode(pred_tokens)
449
+ print(f" Greedy decode (untrained): '{decoded}'")
450
+ print("\n Demo complete!\n")
451
+
452
+
453
+ # ── CLI ──────────────────────────────────────────────────────────────────
454
+
455
+ def main() -> None:
456
+ parser = argparse.ArgumentParser(description="Grassmann Manifold Audio Transcription")
457
+ sub = parser.add_subparsers(dest="command")
458
+
459
+ # train
460
+ p = sub.add_parser("train", help="Train model (GPU)")
461
+ p.add_argument("--from-cache", default=None, help="HF Dataset repo with pre-cached mels")
462
+ p.add_argument("--recipe", default=None, help="Multi-dataset recipe (e.g. '1750h')")
463
+ p.add_argument("--dataset", default="openslr/librispeech_asr")
464
+ p.add_argument("--split", default="train.clean.100")
465
+ p.add_argument("--max-samples", type=int, default=None)
466
+ p.add_argument("--max-per-source", type=int, default=None)
467
+ p.add_argument("--epochs", type=int, default=50)
468
+ p.add_argument("--batch-size", type=int, default=32)
469
+ p.add_argument("--save-dir", default="checkpoints")
470
+ p.add_argument("--run-name", default=None, help="Prefix for uploaded artifacts (e.g. 'ablation-smol')")
471
+ p.add_argument("--decode-every", type=int, default=5)
472
+ _add_model_args(p)
473
+
474
+ # cache
475
+ p = sub.add_parser("cache", help="Cache dataset mels to HF Hub (CPU)")
476
+ p.add_argument("--recipe", required=True)
477
+ p.add_argument("--repo", required=True, help="HF Dataset repo")
478
+ p.add_argument("--cache-dir", default="mel_cache")
479
+ p.add_argument("--max-per-source", type=int, default=None)
480
+ p.add_argument("--workers", type=int, default=0, help="Parallel mel workers (0=sequential, default)")
481
+
482
+ # export
483
+ p = sub.add_parser("export", help="Export model to ONNX")
484
+ p.add_argument("--model", required=True)
485
+ p.add_argument("--output", default="onnx_export")
486
+ p.add_argument("--opset", type=int, default=18)
487
+ p.add_argument("--verify", action="store_true")
488
+
489
+ # eval
490
+ p = sub.add_parser("eval", help="Evaluate WER/CER on LibriSpeech")
491
+ p.add_argument("--model", default="eval_model/best_model.pt")
492
+ p.add_argument("--config", default="eval_model/config.json")
493
+ p.add_argument("--split", default="test.clean")
494
+ p.add_argument("--max-samples", type=int, default=None)
495
+ p.add_argument("--show-errors", type=int, default=10)
496
+
497
+ # transcribe
498
+ p = sub.add_parser("transcribe", help="Transcribe audio file")
499
+ p.add_argument("--model", required=True)
500
+ p.add_argument("--audio", required=True)
501
+
502
+ # demo
503
+ sub.add_parser("demo", help="Sanity check with synthetic data")
504
+
505
+ args = parser.parse_args()
506
+ commands = {
507
+ "train": cmd_train, "cache": cmd_cache, "export": cmd_export,
508
+ "eval": cmd_eval, "transcribe": cmd_transcribe, "demo": lambda a: cmd_demo(),
509
+ }
510
+ if args.command in commands:
511
+ commands[args.command](args)
512
+ else:
513
+ parser.print_help()
514
+ sys.exit(1)
515
+
516
+
517
+ if __name__ == "__main__":
518
+ main()
recipes.yaml ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dataset recipes for training.
2
+ # Each recipe is a list of HuggingFace dataset sources.
3
+ # Use with: python app.py train --recipe <name>
4
+ #
5
+ # Fields per source:
6
+ # name: HuggingFace dataset ID (required)
7
+ # split: Dataset split name (required)
8
+ # config: Dataset config/subset (optional)
9
+ # max_samples: Cap samples from this source (optional)
10
+
11
+ schemas:
12
+ openslr/librispeech_asr:
13
+ audio: audio
14
+ text: text
15
+ mozilla-foundation/common_voice:
16
+ audio: audio
17
+ text: sentence
18
+ google/fleurs:
19
+ audio: audio
20
+ text: transcription
21
+ speechcolab/gigaspeech:
22
+ audio: audio
23
+ text: text
24
+ facebook/voxpopuli:
25
+ audio: audio
26
+ text: normalized_text
27
+ MLCommons/peoples_speech:
28
+ audio: audio
29
+ text: text
30
+ espnet/yodas:
31
+ audio: audio
32
+ text: text
33
+
34
+ recipes:
35
+ librispeech-100:
36
+ - name: openslr/librispeech_asr
37
+ split: train.clean.100
38
+
39
+ librispeech-960:
40
+ - name: openslr/librispeech_asr
41
+ split: train.clean.100
42
+ - name: openslr/librispeech_asr
43
+ split: train.clean.360
44
+ - name: openslr/librispeech_asr
45
+ split: train.other.500
46
+
47
+ 1750h:
48
+ - name: openslr/librispeech_asr
49
+ split: train.clean.100
50
+ - name: openslr/librispeech_asr
51
+ split: train.clean.360
52
+ - name: openslr/librispeech_asr
53
+ split: train.other.500
54
+ - name: facebook/voxpopuli
55
+ config: en
56
+ split: train
57
+ - name: MLCommons/peoples_speech
58
+ config: clean
59
+ split: train
60
+ max_samples: 30000 # ~250h subset of 30K hours
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ torch>=2.0
2
+ torchaudio>=2.0
3
+ datasets>=3.0
4
+ soundfile>=0.12
5
+ torchcodec>=0.1
6
+ sqids>=0.4
7
+ pyyaml>=6.0
src/__init__.py ADDED
File without changes
src/controllers/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .model_controller import ModelController
2
+ from .pipeline_controller import PipelineController
src/controllers/model_controller.py ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ModelController: fluent API for model lifecycle.
3
+
4
+ Provides the chainable interface:
5
+ _model.build().train(...).test(...).save(...)
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, Self
11
+
12
+ import torch
13
+
14
+ from src.interfaces import IModel
15
+ from src.models.config import GrassmannConfig
16
+ from src.models.grassmann_model import GrassmannTranscriptionModel
17
+ from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
18
+ from src.services.tokenizer_service import CharTokenizer
19
+ from src.services.training_service import TrainingService
20
+ from src.services.seq2seq_training_service import Seq2SeqTrainingService
21
+ from src.views.console_view import ConsoleView
22
+
23
+
24
+ class ModelController(IModel):
25
+ """Orchestrates model building, training, evaluation, and inference."""
26
+
27
+ def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer):
28
+ self.config = config
29
+ self.tokenizer = tokenizer
30
+ self.view = ConsoleView()
31
+ self.nn_model: GrassmannTranscriptionModel | GrassmannSeq2SeqModel | None = None
32
+ self.trainer: TrainingService | Seq2SeqTrainingService | None = None
33
+
34
+ def build(self) -> Self:
35
+ self.config.vocab_size = self.tokenizer.vocab_size
36
+
37
+ if self.config.use_seq2seq:
38
+ self.nn_model = GrassmannSeq2SeqModel(self.config)
39
+ self.trainer = Seq2SeqTrainingService(self.nn_model, self.config)
40
+ else:
41
+ self.nn_model = GrassmannTranscriptionModel(self.config)
42
+ self.trainer = TrainingService(self.nn_model, self.config)
43
+
44
+ param_count = sum(p.numel() for p in self.nn_model.parameters())
45
+ self.view.show_model_info(self.config, param_count)
46
+ return self
47
+
48
+ def train(self, train_loader: Any, val_loader: Any, epochs: int | None = None) -> Self:
49
+ assert self.trainer is not None, "Call build() first"
50
+ epochs = epochs or self.config.epochs
51
+
52
+ history = self.trainer.train_loop(
53
+ train_loader, val_loader, epochs, callback=self.view
54
+ )
55
+ self.view.show_training_complete(history)
56
+ return self
57
+
58
+ def test(self, test_loader: Any) -> dict:
59
+ assert self.trainer is not None, "Call build() and train() first"
60
+ test_loss = self.trainer.validate(test_loader)
61
+ results = {"test_loss": test_loss}
62
+ self.view.show_test_results(results)
63
+ return results
64
+
65
+ def fit(self, dataset: Any) -> Self:
66
+ """Convenience: build + train + validate."""
67
+ self.build()
68
+
69
+ train_loader = dataset.get_train_loader(self.config.batch_size)
70
+ val_loader = dataset.get_val_loader(self.config.batch_size)
71
+
72
+ self.train(train_loader, val_loader)
73
+ return self
74
+
75
+ def predict(self, audio: torch.Tensor) -> str:
76
+ assert self.trainer is not None, "Call build() and train()/load() first"
77
+ pred_tokens = self.trainer.greedy_decode(audio)
78
+ text = self.tokenizer.decode(pred_tokens)
79
+ return text
80
+
81
+ def save(self, path: str) -> Self:
82
+ assert self.nn_model is not None
83
+ torch.save(self.nn_model.state_dict(), path)
84
+ self.view.show_message(f"Model saved to {path}")
85
+ return self
86
+
87
+ def load(self, path: str) -> Self:
88
+ if self.nn_model is None:
89
+ self.build()
90
+ assert self.nn_model is not None
91
+ self.nn_model.load_state_dict(torch.load(path, weights_only=True))
92
+ self.view.show_message(f"Model loaded from {path}")
93
+ return self
src/controllers/pipeline_controller.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ PipelineController: end-to-end orchestration.
3
+
4
+ Wires together dataset, model, and services into a single fluent pipeline.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Self
10
+
11
+ from src.interfaces import IDataset
12
+ from src.models.config import GrassmannConfig
13
+ from src.services.audio_service import AudioService
14
+ from src.services.dataset_service import DatasetService
15
+ from src.services.tokenizer_service import CharTokenizer
16
+ from src.repositories.dataset_repository import DatasetRepository
17
+ from src.repositories.cache_repository import CacheRepository
18
+ from src.controllers.model_controller import ModelController
19
+
20
+
21
+ class PipelineController:
22
+ """
23
+ Top-level controller for the full transcription pipeline.
24
+
25
+ Usage (local files):
26
+ pipeline = PipelineController(config)
27
+ pipeline.prepare_dataset("path/to/data")
28
+ .build_model()
29
+ .train()
30
+ .test()
31
+ .transcribe("path/to/audio.wav")
32
+
33
+ Usage (HuggingFace):
34
+ pipeline = PipelineController(config)
35
+ pipeline.prepare_hf_dataset("openslr/librispeech_asr",
36
+ split="train.clean.100")
37
+ .build_model()
38
+ .train()
39
+ .test()
40
+ """
41
+
42
+ def __init__(self, config: GrassmannConfig | None = None,
43
+ cache_dir: str = "checkpoints/mel_cache"):
44
+ self.config = config or GrassmannConfig()
45
+ self.tokenizer = CharTokenizer()
46
+ self.audio_service = AudioService(self.config)
47
+ self._repo = DatasetRepository(self.config, self.tokenizer)
48
+ self._cache = CacheRepository(cache_dir)
49
+ self.dataset: DatasetService | None = None
50
+ self.model: ModelController | None = None
51
+
52
+ def prepare_hf_dataset(
53
+ self,
54
+ dataset_name: str,
55
+ config_name: str | None = None,
56
+ split: str | None = None,
57
+ max_samples: int | None = None,
58
+ ) -> Self:
59
+ """Load a dataset from HuggingFace Hub."""
60
+ self.dataset = DatasetService(self._repo, self._cache)
61
+ self.dataset.load_dataset(
62
+ dataset_name, split=split,
63
+ config_name=config_name,
64
+ max_samples=max_samples,
65
+ ).clean().preprocess().split()
66
+ return self
67
+
68
+ def build_model(self) -> Self:
69
+ self.model = ModelController(self.config, self.tokenizer)
70
+ self.model.build()
71
+ return self
72
+
73
+ def train(self, epochs: int | None = None) -> Self:
74
+ assert self.model is not None and self.dataset is not None
75
+ train_loader = self.dataset.get_train_loader(self.config.batch_size)
76
+ val_loader = self.dataset.get_val_loader(self.config.batch_size)
77
+ self.model.train(train_loader, val_loader, epochs)
78
+ return self
79
+
80
+ def test(self) -> dict:
81
+ assert self.model is not None and self.dataset is not None
82
+ test_loader = self.dataset.get_test_loader(self.config.batch_size)
83
+ return self.model.test(test_loader)
84
+
85
+ def transcribe(self, audio_path: str) -> str:
86
+ assert self.model is not None
87
+ waveform = self.audio_service.load(audio_path)
88
+ mel = self.audio_service.extract_features(waveform)
89
+ mel = self.audio_service.normalize(mel)
90
+ return self.model.predict(mel)
91
+
92
+ def save_model(self, path: str) -> Self:
93
+ assert self.model is not None
94
+ self.model.save(path)
95
+ return self
96
+
97
+ def load_model(self, path: str) -> Self:
98
+ if self.model is None:
99
+ self.build_model()
100
+ assert self.model is not None
101
+ self.model.load(path)
102
+ return self
src/interfaces/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ from .i_dataset import IDataset
2
+ from .i_model import IModel
3
+ from .i_audio import IAudioProcessor
4
+ from .i_tokenizer import ITokenizer
src/interfaces/i_audio.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Any
3
+
4
+ import torch
5
+
6
+
7
+ class IAudioProcessor(ABC):
8
+ """Interface for audio feature extraction."""
9
+
10
+ @abstractmethod
11
+ def load(self, path: str) -> torch.Tensor:
12
+ """Load raw audio from file. Returns waveform tensor."""
13
+ ...
14
+
15
+ @abstractmethod
16
+ def extract_features(self, waveform: torch.Tensor) -> torch.Tensor:
17
+ """Extract features (e.g. mel spectrogram) from waveform."""
18
+ ...
19
+
20
+ @abstractmethod
21
+ def normalize(self, features: torch.Tensor) -> torch.Tensor:
22
+ """Normalize features for model input."""
23
+ ...
src/interfaces/i_dataset.py ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Self
3
+
4
+ from torch.utils.data import DataLoader
5
+
6
+
7
+ class IDataset(ABC):
8
+ """Interface for dataset loading and preprocessing."""
9
+
10
+ @abstractmethod
11
+ def load(self, path: str) -> Self:
12
+ """Load dataset from path."""
13
+ ...
14
+
15
+ @abstractmethod
16
+ def clean(self) -> Self:
17
+ """Clean and filter the dataset."""
18
+ ...
19
+
20
+ @abstractmethod
21
+ def preprocess(self) -> Self:
22
+ """Preprocess dataset (extract features, tokenize labels)."""
23
+ ...
24
+
25
+ @abstractmethod
26
+ def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1) -> Self:
27
+ """Split into train/val/test sets."""
28
+ ...
29
+
30
+ @abstractmethod
31
+ def get_train_loader(self, batch_size: int) -> DataLoader:
32
+ ...
33
+
34
+ @abstractmethod
35
+ def get_val_loader(self, batch_size: int) -> DataLoader:
36
+ ...
37
+
38
+ @abstractmethod
39
+ def get_test_loader(self, batch_size: int) -> DataLoader:
40
+ ...
src/interfaces/i_model.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+ from typing import Any, Self
3
+
4
+ import torch
5
+
6
+
7
+ class IModel(ABC):
8
+ """Interface for the trainable model."""
9
+
10
+ @abstractmethod
11
+ def build(self) -> Self:
12
+ """Build / initialize the model architecture."""
13
+ ...
14
+
15
+ @abstractmethod
16
+ def train(self, train_loader: Any, val_loader: Any, epochs: int) -> Self:
17
+ """Train the model."""
18
+ ...
19
+
20
+ @abstractmethod
21
+ def test(self, test_loader: Any) -> dict:
22
+ """Evaluate on test set. Returns metrics dict."""
23
+ ...
24
+
25
+ @abstractmethod
26
+ def fit(self, dataset: Any) -> Self:
27
+ """Convenience: preprocess + train + validate in one call."""
28
+ ...
29
+
30
+ @abstractmethod
31
+ def predict(self, audio: torch.Tensor) -> str:
32
+ """Transcribe audio tensor to text."""
33
+ ...
34
+
35
+ @abstractmethod
36
+ def save(self, path: str) -> Self:
37
+ """Save model checkpoint."""
38
+ ...
39
+
40
+ @abstractmethod
41
+ def load(self, path: str) -> Self:
42
+ """Load model checkpoint."""
43
+ ...
src/interfaces/i_tokenizer.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from abc import ABC, abstractmethod
2
+
3
+ import torch
4
+
5
+
6
+ class ITokenizer(ABC):
7
+ """Interface for text tokenization (transcription labels)."""
8
+
9
+ @abstractmethod
10
+ def encode(self, text: str) -> torch.Tensor:
11
+ """Encode text string to token indices."""
12
+ ...
13
+
14
+ @abstractmethod
15
+ def decode(self, tokens: torch.Tensor) -> str:
16
+ """Decode token indices back to text string."""
17
+ ...
18
+
19
+ @property
20
+ @abstractmethod
21
+ def vocab_size(self) -> int:
22
+ ...
23
+
24
+ @property
25
+ @abstractmethod
26
+ def blank_id(self) -> int:
27
+ """CTC blank token id."""
28
+ ...
src/models/__init__.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ from .config import GrassmannConfig
2
+ from .grassmann_layer import CausalGrassmannLayer, GrassmannMixer
3
+ from .ablation_layer import AblationLayer, AblationMixer
4
+ from .grassmann_model import GrassmannTranscriptionModel
5
+ from .grassmann_cross_layer import GrassmannCrossLayer
6
+ from .grassmann_decoder import GrassmannDecoder
7
+ from .grassmann_seq2seq import GrassmannSeq2SeqModel
8
+ from .conv_module import ConvModule
9
+ from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
src/models/ablation_layer.py ADDED
@@ -0,0 +1,175 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Ablation Mixing Layer — Symmetric Product Baseline.
3
+
4
+ Drop-in replacement for the Grassmann/Plücker mixer that uses plain
5
+ element-wise symmetric products instead of antisymmetric Plücker
6
+ coordinates. Everything else (linear reduction, multi-scale offsets,
7
+ mean-first anchoring, gated fusion, FFN) stays identical so the only
8
+ variable is whether the Grassmann geometry matters.
9
+
10
+ Matches Dhruv's grassmann-speech implementation:
11
+ - symmetric_products: a_i * b_j + a_j * b_i (vs Plücker: a_i * b_j - a_j * b_i)
12
+ - symmetric_norm_sq: |a|²|b|² + (a·b)² - 2·Σ(aᵢ²bᵢ²) (vs Lagrange: |a|²|b|² - (a·b)²)
13
+ - flip = +1.0 in two-sided mode (vs Plücker: flip = -1.0)
14
+ """
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+
19
+ from .config import GrassmannConfig
20
+
21
+
22
+ class SymmetricEncoder(nn.Module):
23
+ """
24
+ Computes element-wise symmetric products for pairs of reduced vectors.
25
+
26
+ Same index pairs and output dimensionality as PluckerEncoder,
27
+ but uses p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
28
+ instead of the antisymmetric Plücker form.
29
+ """
30
+
31
+ def __init__(self, r: int, eps: float = 1e-8):
32
+ super().__init__()
33
+ self.r = r
34
+ self.eps = eps
35
+ idx_i, idx_j = [], []
36
+ for i in range(r):
37
+ for j in range(i + 1, r):
38
+ idx_i.append(i)
39
+ idx_j.append(j)
40
+ self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
41
+ self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
42
+
43
+ def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
44
+ """
45
+ Compute normalized symmetric products.
46
+
47
+ Args:
48
+ z_t: (batch, seq, r)
49
+ z_delta: (batch, seq, r)
50
+
51
+ Returns:
52
+ (batch, seq, C(r,2))
53
+ """
54
+ # Symmetric: p_{ij} = z_{t,i} * z_{Δ,j} + z_{t,j} * z_{Δ,i}
55
+ p = (
56
+ z_t[..., self.idx_i] * z_delta[..., self.idx_j]
57
+ + z_t[..., self.idx_j] * z_delta[..., self.idx_i]
58
+ )
59
+ norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
60
+ return p / norm
61
+
62
+
63
+ def _symmetric_norm_sq(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
64
+ """
65
+ Squared norm of symmetric products from dot products alone.
66
+ Matches grassmann-speech/plucker.py symmetric_norm_sq:
67
+ |a|²|b|² + (a·b)² - 2·Σ(aᵢ²·bᵢ²)
68
+ """
69
+ aa = (a * a).sum(-1, keepdim=True)
70
+ bb = (b * b).sum(-1, keepdim=True)
71
+ ab = (a * b).sum(-1, keepdim=True)
72
+ return aa * bb + ab * ab - 2 * (a * a * b * b).sum(-1, keepdim=True)
73
+
74
+
75
+ class AblationMixer(nn.Module):
76
+ """
77
+ Symmetric-product mixer — ablation baseline for GrassmannMixer.
78
+
79
+ Identical architecture (linear reduction → multi-scale pairing →
80
+ encoding → mean-first anchoring → projection) but with symmetric
81
+ products replacing Plücker coordinates.
82
+
83
+ Key differences from GrassmannMixer (matching grassmann-speech):
84
+ 1. Symmetric products instead of Plücker coordinates
85
+ 2. symmetric_norm_sq for anchored weighting
86
+ 3. flip = +1.0 in two-sided mode (symmetric products are symmetric
87
+ under swap, so no sign flip — use positive accumulation)
88
+ """
89
+
90
+ def __init__(self, d: int, r: int, offsets: list[int],
91
+ two_sided: bool = False, eps: float = 1e-8):
92
+ super().__init__()
93
+ self.offsets = offsets
94
+ self.two_sided = two_sided
95
+ self.eps = eps
96
+ plucker_dim = r * (r - 1) // 2 # same dimensionality
97
+
98
+ self.W_red = nn.Linear(d, r)
99
+ self.encoder = SymmetricEncoder(r, eps=eps)
100
+ self.W_plu = nn.Linear(plucker_dim, d)
101
+
102
+ def forward(self, h: torch.Tensor) -> torch.Tensor:
103
+ """Mean-first (anchored) form, matching grassmann-speech."""
104
+ B, L, D = h.shape
105
+ z = self.W_red(h)
106
+
107
+ partner = torch.zeros_like(z)
108
+ count = z.new_zeros(1, L, 1)
109
+
110
+ for delta in self.offsets:
111
+ if delta >= L:
112
+ continue
113
+ earlier = z[:, :L - delta, :]
114
+ later = z[:, delta:, :]
115
+
116
+ # Weight by symmetric norm (from dot products, no coordinate expansion)
117
+ w = _symmetric_norm_sq(earlier, later).clamp_min(0).sqrt().clamp_min(self.eps).reciprocal()
118
+
119
+ partner[:, delta:] += w * earlier
120
+ count[:, delta:] += 1
121
+
122
+ if self.two_sided:
123
+ # flip = +1.0 for symmetric (vs -1.0 for Plücker)
124
+ # Symmetric products are invariant under swap(a,b),
125
+ # so no sign flip needed to distinguish direction
126
+ partner[:, :L - delta] += w * later
127
+ count[:, :L - delta] += 1
128
+
129
+ count = count.clamp(min=1)
130
+ avg_partner = partner / count
131
+ p_final = self.encoder(z, avg_partner)
132
+ return self.W_plu(p_final)
133
+
134
+
135
+ class AblationLayer(nn.Module):
136
+ """
137
+ Drop-in replacement for CausalGrassmannLayer using symmetric products.
138
+
139
+ Same gated fusion + FFN, just swaps the mixer.
140
+ """
141
+
142
+ def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
143
+ super().__init__()
144
+ d = config.d_model
145
+ r = config.r_reduced
146
+ two_sided = getattr(config, "two_sided", False)
147
+
148
+ self.mixer = AblationMixer(
149
+ d, r, config.window_offsets,
150
+ two_sided=two_sided, eps=config.plucker_eps,
151
+ )
152
+
153
+ self.W_gate = nn.Linear(2 * d, d)
154
+ self.norm_mix = nn.LayerNorm(d)
155
+ self.dropout_mix = nn.Dropout(config.dropout)
156
+
157
+ self.ffn = nn.Sequential(
158
+ nn.Linear(d, config.d_ff),
159
+ nn.GELU(),
160
+ nn.Dropout(config.dropout),
161
+ nn.Linear(config.d_ff, d),
162
+ nn.Dropout(config.dropout),
163
+ )
164
+ self.norm_ffn = nn.LayerNorm(d)
165
+
166
+ def forward(self, h: torch.Tensor) -> torch.Tensor:
167
+ g = self.mixer(h)
168
+
169
+ u = torch.cat([h, g], dim=-1)
170
+ alpha = torch.sigmoid(self.W_gate(u))
171
+ h_mix = h + (1 - alpha) * (g - h)
172
+
173
+ h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
174
+ h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
175
+ return h_out
src/models/config.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from dataclasses import dataclass, field
2
+
3
+
4
+ @dataclass
5
+ class GrassmannConfig:
6
+ """Configuration for the Grassmann Transcription Model."""
7
+
8
+ # Audio features
9
+ n_mels: int = 80
10
+ sample_rate: int = 16000
11
+ n_fft: int = 400
12
+ hop_length: int = 160
13
+
14
+ # Model architecture
15
+ d_model: int = 256
16
+ r_reduced: int = 32
17
+ n_layers: int = 6
18
+ d_ff: int = 1024 # 4 * d_model
19
+ dropout: float = 0.1
20
+
21
+ # Decoder architecture
22
+ n_decoder_layers: int = 4
23
+ cross_window: int = 8 # encoder positions per side for cross-mixing
24
+
25
+ # Grassmann-specific
26
+ window_offsets: list[int] = field(default_factory=lambda: [1, 2, 4, 8, 12, 16])
27
+ plucker_eps: float = 1e-8
28
+ two_sided: bool = False # True for CTC/encoder (bidirectional)
29
+
30
+ # Conformer options
31
+ use_conformer: bool = False # True = Conformer blocks, False = plain Grassmann
32
+ conv_kernel_size: int = 31 # ConvModule kernel size
33
+ conv_channels: int = 256 # Subsampling conv channels
34
+
35
+ # Ablation: swap Plücker for symmetric products to test if geometry matters
36
+ use_ablation: bool = False
37
+
38
+ # Training
39
+ learning_rate: float = 3e-4
40
+ weight_decay: float = 1e-2
41
+ batch_size: int = 16
42
+ epochs: int = 30
43
+ max_seq_len: int = 2048 # LibriSpeech clips up to ~30s → ~750 frames after subsampling
44
+
45
+ # Architecture mode
46
+ use_seq2seq: bool = True # True = encoder-decoder, False = CTC-only
47
+
48
+ # Tokenizer
49
+ tokenizer_type: str = "char" # "char" or "bpe"
50
+ bpe_vocab_size: int = 1000 # BPE vocab size if tokenizer_type="bpe"
51
+
52
+ # Vocab (set dynamically by tokenizer)
53
+ vocab_size: int = 29 # 26 letters + space + apostrophe + blank
54
+
55
+ @property
56
+ def plucker_dim(self) -> int:
57
+ """Dimension of Plücker coordinate vector: C(r, 2)."""
58
+ return self.r_reduced * (self.r_reduced - 1) // 2
src/models/conformer_block.py ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Conformer Block with Grassmann Mixer.
3
+
4
+ Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100)
5
+ with the self-attention slot replaced by a Grassmann mixer:
6
+
7
+ x → +0.5*FFN → +Mixer → +Conv → +0.5*FFN → LayerNorm
8
+
9
+ This gives the best of both worlds:
10
+ - Grassmann mixer for sequence-level subspace geometry
11
+ - ConvModule for local temporal pattern capture
12
+ - Macaron FFN sandwich for nonlinear capacity
13
+ """
14
+
15
+ import torch
16
+ import torch.nn as nn
17
+
18
+ from .config import GrassmannConfig
19
+ from .conv_module import ConvModule
20
+ from .grassmann_layer import GrassmannMixer
21
+ from .ablation_layer import AblationMixer
22
+
23
+
24
+ def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential:
25
+ """Conformer-style FFN: LayerNorm → Linear → SiLU → Dropout → Linear → Dropout."""
26
+ return nn.Sequential(
27
+ nn.LayerNorm(d),
28
+ nn.Linear(d, d_ff),
29
+ nn.SiLU(),
30
+ nn.Dropout(dropout),
31
+ nn.Linear(d_ff, d),
32
+ nn.Dropout(dropout),
33
+ )
34
+
35
+
36
+ class GatedGrassmannMixer(nn.Module):
37
+ """
38
+ Two-sided Grassmann mix with sigmoid gate as a residual branch.
39
+ (1 - α)(g - u) formulation from Dhruv's implementation.
40
+ """
41
+
42
+ def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
43
+ super().__init__()
44
+ self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps)
45
+ self.gate = nn.Linear(2 * d, d)
46
+
47
+ def forward(self, u: torch.Tensor) -> torch.Tensor:
48
+ g = self.mixer(u)
49
+ alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
50
+ return (1 - alpha) * (g - u)
51
+
52
+
53
+ class GatedAblationMixer(nn.Module):
54
+ """
55
+ Ablation baseline: symmetric-product mixer with the same gate.
56
+ Drop-in replacement for GatedGrassmannMixer.
57
+ """
58
+
59
+ def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
60
+ super().__init__()
61
+ self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps)
62
+ self.gate = nn.Linear(2 * d, d)
63
+
64
+ def forward(self, u: torch.Tensor) -> torch.Tensor:
65
+ g = self.mixer(u)
66
+ alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
67
+ return (1 - alpha) * (g - u)
68
+
69
+
70
+ class GrassmannConformerBlock(nn.Module):
71
+ """
72
+ Conformer block with Grassmann mixer replacing self-attention.
73
+
74
+ Architecture (Macaron block):
75
+ x → +0.5*FFN1 → +GatedGrassmannMixer → +ConvModule → +0.5*FFN2 → LayerNorm
76
+ """
77
+
78
+ def __init__(self, config: GrassmannConfig):
79
+ super().__init__()
80
+ d = config.d_model
81
+ d_ff = config.d_ff
82
+ r = config.r_reduced
83
+ dropout = config.dropout
84
+ kernel = getattr(config, "conv_kernel_size", 31)
85
+ offsets = config.window_offsets
86
+
87
+ # Macaron FFN sandwich
88
+ self.ff1 = _macaron_ffn(d, d_ff, dropout)
89
+ self.ff2 = _macaron_ffn(d, d_ff, dropout)
90
+
91
+ # Grassmann mixer (replaces self-attention)
92
+ self.ln_mix = nn.LayerNorm(d)
93
+ self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps)
94
+ self.drop = nn.Dropout(dropout)
95
+
96
+ # ConvModule for local patterns
97
+ self.conv = ConvModule(d, kernel, dropout)
98
+
99
+ # Final LayerNorm
100
+ self.ln_out = nn.LayerNorm(d)
101
+
102
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
103
+ """
104
+ Args:
105
+ x: (batch, seq_len, d_model)
106
+ Returns:
107
+ (batch, seq_len, d_model)
108
+ """
109
+ # Half-step FFN 1
110
+ x = x + 0.5 * self.ff1(x)
111
+ # Grassmann mixer (with gate as residual)
112
+ x = x + self.drop(self.mixer(self.ln_mix(x)))
113
+ # ConvModule
114
+ x = x + self.conv(x)
115
+ # Half-step FFN 2 + final norm
116
+ return self.ln_out(x + 0.5 * self.ff2(x))
117
+
118
+
119
+ class AblationConformerBlock(nn.Module):
120
+ """
121
+ Conformer block with symmetric-product mixer (ablation baseline).
122
+ Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products.
123
+ """
124
+
125
+ def __init__(self, config: GrassmannConfig):
126
+ super().__init__()
127
+ d = config.d_model
128
+ d_ff = config.d_ff
129
+ r = config.r_reduced
130
+ dropout = config.dropout
131
+ kernel = getattr(config, "conv_kernel_size", 31)
132
+ offsets = config.window_offsets
133
+
134
+ self.ff1 = _macaron_ffn(d, d_ff, dropout)
135
+ self.ff2 = _macaron_ffn(d, d_ff, dropout)
136
+
137
+ self.ln_mix = nn.LayerNorm(d)
138
+ self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps)
139
+ self.drop = nn.Dropout(dropout)
140
+
141
+ self.conv = ConvModule(d, kernel, dropout)
142
+ self.ln_out = nn.LayerNorm(d)
143
+
144
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
145
+ x = x + 0.5 * self.ff1(x)
146
+ x = x + self.drop(self.mixer(self.ln_mix(x)))
147
+ x = x + self.conv(x)
148
+ return self.ln_out(x + 0.5 * self.ff2(x))
src/models/conv_module.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Conformer ConvModule.
3
+
4
+ Captures local temporal patterns that Grassmann mixing misses.
5
+ From Gulati et al. 2020 (arXiv 2005.08100):
6
+
7
+ LayerNorm → Pointwise Conv (GLU) → Depthwise Conv → BatchNorm → SiLU → Pointwise Conv → Dropout
8
+
9
+ This is the key component that makes Conformer superior to pure
10
+ Transformer/Grassmann for speech — it captures local phonetic patterns
11
+ within a convolutional receptive field.
12
+ """
13
+
14
+ import torch
15
+ import torch.nn as nn
16
+ import torch.nn.functional as F
17
+
18
+
19
+ class ConvModule(nn.Module):
20
+ """Conformer convolution module with depthwise separable convolution."""
21
+
22
+ def __init__(self, d_model: int, kernel_size: int = 31, dropout: float = 0.1):
23
+ super().__init__()
24
+ self.ln = nn.LayerNorm(d_model)
25
+ # Pointwise expansion with GLU gating
26
+ self.pw1 = nn.Conv1d(d_model, 2 * d_model, kernel_size=1)
27
+ # Depthwise convolution (each channel independently)
28
+ self.dw = nn.Conv1d(
29
+ d_model, d_model, kernel_size,
30
+ padding=kernel_size // 2, groups=d_model,
31
+ )
32
+ self.bn = nn.BatchNorm1d(d_model)
33
+ # Pointwise projection back
34
+ self.pw2 = nn.Conv1d(d_model, d_model, kernel_size=1)
35
+ self.drop = nn.Dropout(dropout)
36
+
37
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
38
+ """
39
+ Args:
40
+ x: (batch, seq_len, d_model)
41
+ Returns:
42
+ (batch, seq_len, d_model)
43
+ """
44
+ # LayerNorm → transpose for Conv1d
45
+ y = self.ln(x).transpose(1, 2) # (B, d, T)
46
+ # Pointwise expansion + GLU gate
47
+ y = F.glu(self.pw1(y), dim=1) # (B, d, T)
48
+ # Depthwise conv + BatchNorm + SiLU
49
+ y = F.silu(self.bn(self.dw(y))) # (B, d, T)
50
+ # Pointwise projection + dropout
51
+ y = self.drop(self.pw2(y)) # (B, d, T)
52
+ return y.transpose(1, 2) # (B, T, d)
src/models/grassmann_cross_layer.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Grassmann Cross-Mixing Layer.
3
+
4
+ Replaces cross-attention in the decoder with Grassmann geometry.
5
+ Instead of Q*K^T dot products between decoder and encoder states,
6
+ we form Plücker coordinates between decoder positions and encoder
7
+ positions within a local monotonic window.
8
+
9
+ This is the key novel component — cross-sequence interaction via
10
+ subspace geometry on Gr(2, r) rather than attention weights.
11
+ """
12
+
13
+ import torch
14
+ import torch.nn as nn
15
+
16
+ from .grassmann_layer import PluckerEncoder
17
+ from .config import GrassmannConfig
18
+
19
+
20
+ class GrassmannCrossLayer(nn.Module):
21
+ """
22
+ Cross-mixing between decoder and encoder via Grassmann/Plücker geometry.
23
+
24
+ For each decoder position t, estimates an alignment position in the
25
+ encoder sequence, then computes Plücker features between the decoder
26
+ state and encoder states in a window around that alignment.
27
+
28
+ This replaces cross-attention entirely — no Q*K^T, no softmax,
29
+ just subspace geometry on a manifold.
30
+ """
31
+
32
+ def __init__(self, config: GrassmannConfig):
33
+ super().__init__()
34
+ d = config.d_model
35
+ r = config.r_reduced
36
+ plucker_dim = config.plucker_dim
37
+
38
+ # Separate reduction projections for decoder and encoder
39
+ self.W_red_dec = nn.Linear(d, r)
40
+ self.W_red_enc = nn.Linear(d, r)
41
+
42
+ # Plücker encoder (shared geometry)
43
+ self.plucker = PluckerEncoder(r, eps=config.plucker_eps)
44
+
45
+ # Project Plücker features back to model dim
46
+ self.W_plu = nn.Linear(plucker_dim, d)
47
+
48
+ # Gated fusion: blend decoder state with cross-mixed features
49
+ self.W_gate = nn.Linear(2 * d, d)
50
+
51
+ # Layer norm
52
+ self.norm = nn.LayerNorm(d)
53
+ self.dropout = nn.Dropout(config.dropout)
54
+
55
+ # Cross-mixing window: how many encoder positions to look at
56
+ # per decoder position (on each side of the alignment point)
57
+ self.cross_window = config.cross_window
58
+
59
+ def _estimate_alignment(
60
+ self, dec_len: int, enc_len: int, device: torch.device
61
+ ) -> torch.Tensor:
62
+ """
63
+ Estimate monotonic alignment: decoder position t maps to
64
+ encoder position t * (enc_len / dec_len).
65
+
66
+ Returns integer encoder indices for each decoder position.
67
+ """
68
+ # Linear interpolation: dec pos 0..T_dec maps to enc pos 0..T_enc
69
+ dec_positions = torch.arange(dec_len, device=device, dtype=torch.float)
70
+ scale = enc_len / max(dec_len, 1)
71
+ enc_positions = (dec_positions * scale).long().clamp(0, enc_len - 1)
72
+ return enc_positions
73
+
74
+ def forward(
75
+ self, h_dec: torch.Tensor, h_enc: torch.Tensor
76
+ ) -> torch.Tensor:
77
+ """
78
+ Args:
79
+ h_dec: (batch, T_dec, d_model) - decoder hidden states
80
+ h_enc: (batch, T_enc, d_model) - encoder hidden states
81
+
82
+ Returns:
83
+ h_out: (batch, T_dec, d_model) - cross-mixed decoder states
84
+ """
85
+ B, T_dec, D = h_dec.shape
86
+ T_enc = h_enc.shape[1]
87
+
88
+ # Reduce both to Grassmann space
89
+ z_dec = self.W_red_dec(h_dec) # (B, T_dec, r)
90
+ z_enc = self.W_red_enc(h_enc) # (B, T_enc, r)
91
+
92
+ # Estimate alignment positions
93
+ align = self._estimate_alignment(T_dec, T_enc, h_dec.device) # (T_dec,)
94
+
95
+ # For each decoder position, gather encoder states in a window
96
+ # around the alignment point and compute Plücker features
97
+ w = self.cross_window
98
+ g_accum = torch.zeros_like(h_dec) # (B, T_dec, D)
99
+ count = 0
100
+
101
+ for offset in range(-w, w + 1):
102
+ # Encoder indices for this offset
103
+ enc_idx = (align + offset).clamp(0, T_enc - 1) # (T_dec,)
104
+
105
+ # Gather encoder states at these positions
106
+ enc_gathered = z_enc[:, enc_idx, :] # (B, T_dec, r)
107
+
108
+ # Plücker encoding between decoder and encoder states
109
+ p = self.plucker(z_dec, enc_gathered) # (B, T_dec, plucker_dim)
110
+
111
+ # Project back
112
+ g = self.W_plu(p) # (B, T_dec, D)
113
+ g_accum += g
114
+ count += 1
115
+
116
+ g_cross = g_accum / max(count, 1)
117
+
118
+ # Gated fusion
119
+ u = torch.cat([h_dec, g_cross], dim=-1) # (B, T_dec, 2D)
120
+ alpha = torch.sigmoid(self.W_gate(u)) # (B, T_dec, D)
121
+ h_mixed = alpha * h_dec + (1 - alpha) * g_cross
122
+
123
+ # Residual + layer norm + dropout
124
+ h_out = self.norm(h_dec + self.dropout(h_mixed))
125
+
126
+ return h_out
src/models/grassmann_decoder.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Grassmann Decoder.
3
+
4
+ A fully attention-free autoregressive decoder using:
5
+ 1. Causal Grassmann self-mixing (same as encoder)
6
+ 2. Grassmann cross-mixing (replaces cross-attention)
7
+ 3. Position-wise FFN
8
+
9
+ Each decoder block: CausalSelfMix → CrossMix → FFN
10
+ Stacked M times, with token embeddings and output projection.
11
+ """
12
+
13
+ import math
14
+
15
+ import torch
16
+ import torch.nn as nn
17
+
18
+ from .config import GrassmannConfig
19
+ from .grassmann_layer import CausalGrassmannLayer
20
+ from .grassmann_cross_layer import GrassmannCrossLayer
21
+
22
+
23
+ class GrassmannDecoderBlock(nn.Module):
24
+ """
25
+ One decoder block: self-mixing → cross-mixing → FFN.
26
+
27
+ The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder).
28
+ The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder).
29
+ The FFN is already included in CausalGrassmannLayer, so we only add
30
+ the cross-mixing step between self-mixing and the FFN residual.
31
+ """
32
+
33
+ def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
34
+ super().__init__()
35
+ d = config.d_model
36
+
37
+ # Causal self-mixing (includes FFN)
38
+ self.self_mix = CausalGrassmannLayer(config, layer_idx)
39
+
40
+ # Grassmann cross-mixing with encoder
41
+ self.cross_mix = GrassmannCrossLayer(config)
42
+
43
+ # Additional FFN after cross-mixing
44
+ self.ffn = nn.Sequential(
45
+ nn.Linear(d, config.d_ff),
46
+ nn.GELU(),
47
+ nn.Dropout(config.dropout),
48
+ nn.Linear(config.d_ff, d),
49
+ nn.Dropout(config.dropout),
50
+ )
51
+ self.norm_ffn = nn.LayerNorm(d)
52
+
53
+ def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor:
54
+ """
55
+ Args:
56
+ h_dec: (batch, T_dec, d_model) - decoder states
57
+ h_enc: (batch, T_enc, d_model) - encoder output
58
+
59
+ Returns:
60
+ (batch, T_dec, d_model) - updated decoder states
61
+ """
62
+ # Self-mixing (causal, within decoder sequence)
63
+ h = self.self_mix(h_dec)
64
+
65
+ # Cross-mixing (decoder ↔ encoder via Grassmann geometry)
66
+ h = self.cross_mix(h, h_enc)
67
+
68
+ # FFN with residual
69
+ h_out = self.norm_ffn(h + self.ffn(h))
70
+
71
+ return h_out
72
+
73
+
74
+ class GrassmannDecoder(nn.Module):
75
+ """
76
+ Full Grassmann decoder: token embeddings → M decoder blocks → output logits.
77
+
78
+ Fully attention-free. All sequence interactions happen through
79
+ Plücker coordinates on Gr(2, r).
80
+ """
81
+
82
+ def __init__(self, config: GrassmannConfig):
83
+ super().__init__()
84
+ self.config = config
85
+
86
+ # Token embeddings
87
+ self.token_embed = nn.Embedding(config.vocab_size, config.d_model)
88
+ self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model)
89
+ self.embed_dropout = nn.Dropout(config.dropout)
90
+ self.embed_scale = math.sqrt(config.d_model)
91
+
92
+ # Decoder blocks
93
+ self.blocks = nn.ModuleList([
94
+ GrassmannDecoderBlock(config, i)
95
+ for i in range(config.n_decoder_layers)
96
+ ])
97
+
98
+ # Output projection to vocab
99
+ self.output_norm = nn.LayerNorm(config.d_model)
100
+ self.output_proj = nn.Linear(config.d_model, config.vocab_size)
101
+
102
+ def forward(
103
+ self,
104
+ tokens: torch.Tensor,
105
+ encoder_out: torch.Tensor,
106
+ ) -> torch.Tensor:
107
+ """
108
+ Args:
109
+ tokens: (batch, T_dec) - input token indices (shifted right)
110
+ encoder_out: (batch, T_enc, d_model) - encoder hidden states
111
+
112
+ Returns:
113
+ logits: (batch, T_dec, vocab_size)
114
+ """
115
+ B, T = tokens.shape
116
+
117
+ # Token + positional embeddings
118
+ positions = torch.arange(T, device=tokens.device).unsqueeze(0)
119
+ h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions)
120
+ h = self.embed_dropout(h)
121
+
122
+ # Decoder blocks
123
+ for block in self.blocks:
124
+ h = block(h, encoder_out)
125
+
126
+ # Output
127
+ h = self.output_norm(h)
128
+ logits = self.output_proj(h)
129
+
130
+ return logits
src/models/grassmann_layer.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Causal Grassmann Mixing Layer.
3
+
4
+ Implements the core architecture from "Attention Is Not What You Need"
5
+ (arXiv:2512.19428). Replaces self-attention with:
6
+
7
+ 1. Linear reduction: h_t ∈ R^d → z_t ∈ R^r
8
+ 2. Multi-scale pairing: (z_t, z_{t+Δ}) for Δ ∈ W
9
+ 3. Plücker encoding: p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i}
10
+ 4. Projection back: g_t = W_plu @ p_hat + b_plu
11
+ 5. Gated fusion: h_mix = α ⊙ h + (1-α) ⊙ g
12
+ 6. Feed-forward block: standard FFN with GELU
13
+ """
14
+
15
+ import torch
16
+ import torch.nn as nn
17
+ import torch.nn.functional as F
18
+
19
+ from .config import GrassmannConfig
20
+
21
+
22
+ class PluckerEncoder(nn.Module):
23
+ """Computes Plücker coordinates for pairs of reduced vectors."""
24
+
25
+ def __init__(self, r: int, eps: float = 1e-8):
26
+ super().__init__()
27
+ self.r = r
28
+ self.eps = eps
29
+ # Precompute index pairs (i, j) where i < j
30
+ idx_i, idx_j = [], []
31
+ for i in range(r):
32
+ for j in range(i + 1, r):
33
+ idx_i.append(i)
34
+ idx_j.append(j)
35
+ self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
36
+ self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))
37
+
38
+ def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
39
+ """
40
+ Compute normalized Plücker coordinates.
41
+
42
+ Args:
43
+ z_t: (batch, seq, r) - reduced states at position t
44
+ z_delta: (batch, seq, r) - reduced states at position t+Δ
45
+
46
+ Returns:
47
+ (batch, seq, C(r,2)) - normalized Plücker vectors
48
+ """
49
+ # p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i}
50
+ p = (
51
+ z_t[..., self.idx_i] * z_delta[..., self.idx_j]
52
+ - z_t[..., self.idx_j] * z_delta[..., self.idx_i]
53
+ )
54
+ # Normalize for numerical stability
55
+ norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
56
+ return p / norm
57
+
58
+
59
+ class GrassmannMixer(nn.Module):
60
+ """
61
+ Grassmann/Plücker mixer — the core sequence interaction.
62
+
63
+ Supports:
64
+ - Causal (one-sided) or bidirectional (two-sided) mixing
65
+ - Mean-first (anchored) form: 3.5x faster, mathematically exact
66
+ - Standard (as-written) form: project then average
67
+
68
+ Two-sided mixing is appropriate for CTC/encoder mode since speech
69
+ isn't generated left-to-right. Antisymmetry of Plücker coordinates
70
+ naturally distinguishes past from future even in two-sided mode.
71
+ """
72
+
73
+ def __init__(self, d: int, r: int, offsets: list[int],
74
+ two_sided: bool = False, eps: float = 1e-8):
75
+ super().__init__()
76
+ self.offsets = offsets
77
+ self.two_sided = two_sided
78
+ self.eps = eps
79
+ plucker_dim = r * (r - 1) // 2
80
+
81
+ self.W_red = nn.Linear(d, r)
82
+ self.plucker = PluckerEncoder(r, eps=eps)
83
+ self.W_plu = nn.Linear(plucker_dim, d)
84
+
85
+ def forward(self, h: torch.Tensor) -> torch.Tensor:
86
+ """
87
+ Mean-first (anchored) form: average partners before projection.
88
+ Mathematically equivalent to as-written, but ~3.5x faster on GPU
89
+ because it does one W_plu matmul instead of one per offset.
90
+ """
91
+ B, L, D = h.shape
92
+ z = self.W_red(h) # (B, L, r)
93
+
94
+ # Accumulate weighted partner vectors, then compute Plücker once
95
+ partner = torch.zeros_like(z) # (B, L, r)
96
+ count = z.new_zeros(1, L, 1)
97
+
98
+ for delta in self.offsets:
99
+ if delta >= L:
100
+ continue
101
+ earlier = z[:, :L - delta, :]
102
+ later = z[:, delta:, :]
103
+
104
+ # Compute norm for weighting (from dot products, numerically stable)
105
+ p = self.plucker(earlier, later)
106
+ w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal()
107
+
108
+ # Assign to later position (causal: t+Δ sees t)
109
+ partner[:, delta:] += w * earlier
110
+ count[:, delta:] += 1
111
+
112
+ if self.two_sided:
113
+ # Also let earlier position see later (bidirectional)
114
+ # Plücker antisymmetry (flip sign) distinguishes direction
115
+ partner[:, :L - delta] += -w * later
116
+ count[:, :L - delta] += 1
117
+
118
+ # Average partners, compute Plücker features, project
119
+ count = count.clamp(min=1)
120
+ avg_partner = partner / count
121
+ # Compute final Plücker between each position and its avg partner
122
+ p_final = self.plucker(z, avg_partner)
123
+ return self.W_plu(p_final)
124
+
125
+
126
+ class CausalGrassmannLayer(nn.Module):
127
+ """
128
+ One layer of the Causal Grassmann Transformer.
129
+
130
+ Replaces the self-attention block with Grassmann/Plücker mixing,
131
+ followed by the standard position-wise FFN.
132
+
133
+ Supports two_sided mode for encoder/CTC use cases.
134
+ """
135
+
136
+ def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
137
+ super().__init__()
138
+ d = config.d_model
139
+ r = config.r_reduced
140
+ two_sided = getattr(config, "two_sided", False)
141
+
142
+ # Grassmann mixer
143
+ self.mixer = GrassmannMixer(
144
+ d, r, config.window_offsets,
145
+ two_sided=two_sided, eps=config.plucker_eps,
146
+ )
147
+
148
+ # Gated fusion
149
+ self.W_gate = nn.Linear(2 * d, d)
150
+
151
+ # Layer norm after mixing
152
+ self.norm_mix = nn.LayerNorm(d)
153
+ self.dropout_mix = nn.Dropout(config.dropout)
154
+
155
+ # Feed-forward block
156
+ self.ffn = nn.Sequential(
157
+ nn.Linear(d, config.d_ff),
158
+ nn.GELU(),
159
+ nn.Dropout(config.dropout),
160
+ nn.Linear(config.d_ff, d),
161
+ nn.Dropout(config.dropout),
162
+ )
163
+ self.norm_ffn = nn.LayerNorm(d)
164
+
165
+ def forward(self, h: torch.Tensor) -> torch.Tensor:
166
+ """
167
+ Args:
168
+ h: (batch, seq_len, d_model)
169
+ Returns:
170
+ h': (batch, seq_len, d_model)
171
+ """
172
+ # Grassmann mixing
173
+ g = self.mixer(h) # (B, L, D)
174
+
175
+ # Gated fusion: (1-α)(g-h) as residual (Dhruv's variant)
176
+ u = torch.cat([h, g], dim=-1)
177
+ alpha = torch.sigmoid(self.W_gate(u))
178
+ h_mix = h + (1 - alpha) * (g - h)
179
+
180
+ # Layer norm + dropout + residual
181
+ h_hat = self.norm_mix(h + self.dropout_mix(h_mix))
182
+
183
+ # Feed-forward with residual
184
+ h_out = self.norm_ffn(h_hat + self.ffn(h_hat))
185
+
186
+ return h_out
src/models/grassmann_model.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Full Grassmann Transcription Model.
3
+
4
+ Audio → Mel Spectrogram features → Grassmann Layers → CTC Output
5
+
6
+ Architecture:
7
+ 1. Conv frontend: projects mel features into d_model
8
+ 2. Positional encoding
9
+ 3. N x CausalGrassmannLayer
10
+ 4. Linear CTC head → vocab_size
11
+ """
12
+
13
+ import math
14
+
15
+ import torch
16
+ import torch.nn as nn
17
+ import torch.nn.functional as F
18
+
19
+ from .config import GrassmannConfig
20
+ from .grassmann_layer import CausalGrassmannLayer
21
+ from .ablation_layer import AblationLayer
22
+ from .conformer_block import GrassmannConformerBlock, AblationConformerBlock
23
+
24
+
25
+ class ConvFrontend(nn.Module):
26
+ """Conv1d feature extractor for mel spectrograms (legacy)."""
27
+
28
+ def __init__(self, n_mels: int, d_model: int):
29
+ super().__init__()
30
+ self.conv = nn.Sequential(
31
+ nn.Conv1d(n_mels, d_model // 2, kernel_size=3, stride=2, padding=1),
32
+ nn.GELU(),
33
+ nn.Conv1d(d_model // 2, d_model, kernel_size=3, stride=2, padding=1),
34
+ nn.GELU(),
35
+ )
36
+
37
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
38
+ out = self.conv(x) # (B, d_model, T')
39
+ return out.transpose(1, 2) # (B, T', d_model)
40
+
41
+
42
+ class Conv2dSubsampling(nn.Module):
43
+ """
44
+ Conv2d subsampling from Conformer (Gulati et al. 2020).
45
+
46
+ Two 3x3 stride-2 convolutions: 10ms fbank frames → 40ms encoder frames.
47
+ Standard ASR frontend used by ESPnet, WeNet, etc.
48
+ """
49
+
50
+ def __init__(self, n_mels: int, channels: int, d_model: int):
51
+ super().__init__()
52
+ self.conv1 = nn.Conv2d(1, channels, kernel_size=3, stride=2)
53
+ self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, stride=2)
54
+ # After two 3x3 stride-2 convs: freq dim = ((n_mels - 1) // 2 - 1) // 2
55
+ freq_out = ((n_mels - 1) // 2 - 1) // 2
56
+ self.out = nn.Linear(channels * freq_out, d_model)
57
+
58
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
59
+ """
60
+ Args:
61
+ x: (batch, n_mels, time) - mel spectrogram
62
+
63
+ Returns:
64
+ (batch, time', d_model)
65
+ """
66
+ # Reshape: (B, n_mels, T) → (B, 1, T, n_mels) for Conv2d
67
+ x = x.transpose(1, 2).unsqueeze(1) # (B, 1, T, n_mels)
68
+ x = F.relu(self.conv1(x))
69
+ x = F.relu(self.conv2(x))
70
+ b, c, t, f = x.shape
71
+ x = x.transpose(1, 2).reshape(b, t, c * f) # (B, T', C*F)
72
+ return self.out(x) # (B, T', d_model)
73
+
74
+
75
+ class PositionalEncoding(nn.Module):
76
+ """Sinusoidal positional encoding."""
77
+
78
+ def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
79
+ super().__init__()
80
+ self.dropout = nn.Dropout(dropout)
81
+
82
+ pe = torch.zeros(max_len, d_model)
83
+ position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
84
+ div_term = torch.exp(
85
+ torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
86
+ )
87
+ pe[:, 0::2] = torch.sin(position * div_term)
88
+ pe[:, 1::2] = torch.cos(position * div_term)
89
+ self.register_buffer("pe", pe.unsqueeze(0)) # (1, max_len, d_model)
90
+
91
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
92
+ x = x + self.pe[:, : x.size(1)]
93
+ return self.dropout(x)
94
+
95
+
96
+ class GrassmannTranscriptionModel(nn.Module):
97
+ """
98
+ Full model: Audio features → Grassmann layers → CTC output.
99
+
100
+ Supports two encoder modes:
101
+ - Plain: CausalGrassmannLayer stack (original)
102
+ - Conformer: GrassmannConformerBlock stack (with ConvModule + Macaron FFN)
103
+
104
+ And two frontend modes:
105
+ - Conv1d (legacy)
106
+ - Conv2d subsampling (proper ASR, from Conformer)
107
+ """
108
+
109
+ def __init__(self, config: GrassmannConfig):
110
+ super().__init__()
111
+ self.config = config
112
+
113
+ # Frontend: mel → d_model
114
+ if config.use_conformer:
115
+ self.frontend = Conv2dSubsampling(
116
+ config.n_mels, config.conv_channels, config.d_model
117
+ )
118
+ else:
119
+ self.frontend = ConvFrontend(config.n_mels, config.d_model)
120
+
121
+ # Positional encoding
122
+ self.pos_enc = PositionalEncoding(
123
+ config.d_model, config.max_seq_len, config.dropout
124
+ )
125
+
126
+ # Encoder layers
127
+ ablation = getattr(config, "use_ablation", False)
128
+ if config.use_conformer:
129
+ block_cls = AblationConformerBlock if ablation else GrassmannConformerBlock
130
+ self.layers = nn.ModuleList(
131
+ [block_cls(config) for _ in range(config.n_layers)]
132
+ )
133
+ else:
134
+ layer_cls = AblationLayer if ablation else CausalGrassmannLayer
135
+ self.layers = nn.ModuleList(
136
+ [layer_cls(config, i) for i in range(config.n_layers)]
137
+ )
138
+
139
+ # CTC output head
140
+ self.output_proj = nn.Linear(config.d_model, config.vocab_size)
141
+
142
+ def forward(self, mel: torch.Tensor) -> torch.Tensor:
143
+ """
144
+ Args:
145
+ mel: (batch, n_mels, time) - mel spectrogram
146
+
147
+ Returns:
148
+ log_probs: (time', batch, vocab_size) - CTC-formatted log probs
149
+ """
150
+ # Conv frontend
151
+ x = self.frontend(mel) # (B, T', d_model)
152
+
153
+ # Positional encoding
154
+ x = self.pos_enc(x)
155
+
156
+ # Grassmann layers
157
+ for layer in self.layers:
158
+ x = layer(x)
159
+
160
+ # CTC head
161
+ logits = self.output_proj(x) # (B, T', vocab_size)
162
+ log_probs = logits.log_softmax(dim=-1)
163
+
164
+ # CTC expects (T, B, C)
165
+ return log_probs.transpose(0, 1)
166
+
167
+ def get_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
168
+ """Calculate output sequence lengths after conv downsampling."""
169
+ lengths = input_lengths
170
+ if self.config.use_conformer:
171
+ # Conv2d subsampling: two 3x3 stride-2 convs (no padding)
172
+ # Each conv: out = (in - 3) // 2 + 1 = (in - 1) // 2
173
+ for _ in range(2):
174
+ lengths = (lengths - 1) // 2
175
+ else:
176
+ # Conv1d frontend: two stride-2 convs with padding=1
177
+ for _ in range(2):
178
+ lengths = (lengths + 1) // 2
179
+ return lengths
src/models/grassmann_seq2seq.py ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Grassmann Seq2Seq Model — Fully Attention-Free Encoder-Decoder.
3
+
4
+ Audio → Grassmann Encoder → Grassmann Decoder → Text
5
+
6
+ Every sequence interaction in this model happens through Plücker
7
+ coordinates on the Grassmann manifold Gr(2, r). There are zero
8
+ attention matrices anywhere in the architecture:
9
+
10
+ Encoder self-mixing: Plücker(z_t, z_{t+Δ}) — local causal windows
11
+ Decoder self-mixing: Plücker(z_t, z_{t+Δ}) — causal autoregressive
12
+ Decoder cross-mixing: Plücker(z_dec, z_enc) — monotonic alignment windows
13
+
14
+ Trained with teacher forcing and cross-entropy loss.
15
+ """
16
+
17
+ import torch
18
+ import torch.nn as nn
19
+
20
+ from .config import GrassmannConfig
21
+ from .grassmann_model import ConvFrontend, PositionalEncoding
22
+ from .grassmann_layer import CausalGrassmannLayer
23
+ from .grassmann_decoder import GrassmannDecoder
24
+
25
+
26
+ class GrassmannEncoder(nn.Module):
27
+ """Audio encoder: mel spectrogram → Grassmann layers → encoder states."""
28
+
29
+ def __init__(self, config: GrassmannConfig):
30
+ super().__init__()
31
+ self.frontend = ConvFrontend(config.n_mels, config.d_model)
32
+ self.pos_enc = PositionalEncoding(
33
+ config.d_model, config.max_seq_len, config.dropout
34
+ )
35
+ self.layers = nn.ModuleList([
36
+ CausalGrassmannLayer(config, i) for i in range(config.n_layers)
37
+ ])
38
+
39
+ def forward(self, mel: torch.Tensor) -> torch.Tensor:
40
+ """
41
+ Args:
42
+ mel: (batch, n_mels, time) - mel spectrogram
43
+
44
+ Returns:
45
+ (batch, T_enc, d_model) - encoder hidden states
46
+ """
47
+ x = self.frontend(mel)
48
+ x = self.pos_enc(x)
49
+ for layer in self.layers:
50
+ x = layer(x)
51
+ return x
52
+
53
+
54
+ class GrassmannSeq2SeqModel(nn.Module):
55
+ """
56
+ Fully attention-free encoder-decoder for audio transcription.
57
+
58
+ Architecture:
59
+ Encoder: ConvFrontend → PosEnc → N × CausalGrassmannLayer
60
+ Decoder: TokenEmbed → PosEmbed → M × (CausalGrassmann + CrossGrassmann + FFN)
61
+ Output: Linear → vocab logits
62
+
63
+ All interactions use Plücker coordinates. Zero attention matrices.
64
+ """
65
+
66
+ # Special token IDs
67
+ SOS_ID = 0 # Start of sequence (overloads blank/pad)
68
+ EOS_ID = 0 # End of sequence (same as SOS for simplicity)
69
+ PAD_ID = 0 # Padding
70
+
71
+ def __init__(self, config: GrassmannConfig):
72
+ super().__init__()
73
+ self.config = config
74
+ self.encoder = GrassmannEncoder(config)
75
+ self.decoder = GrassmannDecoder(config)
76
+
77
+ def forward(
78
+ self,
79
+ mel: torch.Tensor,
80
+ target_tokens: torch.Tensor,
81
+ ) -> torch.Tensor:
82
+ """
83
+ Teacher-forced forward pass.
84
+
85
+ Args:
86
+ mel: (batch, n_mels, time) - input mel spectrogram
87
+ target_tokens: (batch, T_target) - target token indices
88
+
89
+ Returns:
90
+ logits: (batch, T_target, vocab_size)
91
+ """
92
+ # Encode audio
93
+ encoder_out = self.encoder(mel) # (B, T_enc, d)
94
+
95
+ # Decode with teacher forcing
96
+ # Input to decoder is target shifted right (prepend SOS)
97
+ dec_input = self._shift_right(target_tokens)
98
+ logits = self.decoder(dec_input, encoder_out)
99
+
100
+ return logits
101
+
102
+ def _shift_right(self, tokens: torch.Tensor) -> torch.Tensor:
103
+ """Shift tokens right and prepend SOS token for teacher forcing."""
104
+ sos = torch.full(
105
+ (tokens.shape[0], 1), self.SOS_ID,
106
+ dtype=tokens.dtype, device=tokens.device
107
+ )
108
+ # Prepend SOS, drop last token
109
+ return torch.cat([sos, tokens[:, :-1]], dim=1)
110
+
111
+ def get_encoder_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
112
+ """Calculate encoder output lengths after conv downsampling."""
113
+ lengths = input_lengths
114
+ for _ in range(2): # 2 conv layers with stride 2
115
+ lengths = (lengths + 1) // 2
116
+ return lengths
117
+
118
+ @torch.no_grad()
119
+ def greedy_decode(
120
+ self,
121
+ mel: torch.Tensor,
122
+ max_len: int = 200,
123
+ eos_id: int = 0,
124
+ repetition_penalty: float = 1.5,
125
+ no_repeat_ngram: int = 3,
126
+ ) -> torch.Tensor:
127
+ """
128
+ Autoregressive greedy decoding with repetition suppression.
129
+
130
+ Args:
131
+ mel: (1, n_mels, time) - single mel spectrogram
132
+ max_len: maximum output length
133
+ eos_id: end-of-sequence token ID
134
+ repetition_penalty: penalize tokens that already appeared (>1.0 to suppress)
135
+ no_repeat_ngram: block any n-gram that already occurred
136
+
137
+ Returns:
138
+ (seq_len,) - predicted token indices
139
+ """
140
+ self.eval()
141
+ device = mel.device
142
+
143
+ # Encode once
144
+ encoder_out = self.encoder(mel) # (1, T_enc, d)
145
+
146
+ # Start with SOS token
147
+ generated = [self.SOS_ID]
148
+
149
+ for _ in range(max_len):
150
+ # Build decoder input from generated tokens so far
151
+ dec_input = torch.tensor(
152
+ [generated], dtype=torch.long, device=device
153
+ ) # (1, t)
154
+
155
+ # Decode
156
+ logits = self.decoder(dec_input, encoder_out) # (1, t, vocab)
157
+ next_logits = logits[0, -1, :] # (vocab,)
158
+
159
+ # Repetition penalty: reduce logits for tokens already generated
160
+ if repetition_penalty != 1.0:
161
+ for prev_token in set(generated):
162
+ if next_logits[prev_token] > 0:
163
+ next_logits[prev_token] /= repetition_penalty
164
+ else:
165
+ next_logits[prev_token] *= repetition_penalty
166
+
167
+ # No-repeat n-gram blocking
168
+ if no_repeat_ngram > 0 and len(generated) >= no_repeat_ngram:
169
+ # Check if any token would create a repeated n-gram
170
+ ngram_prefix = generated[-(no_repeat_ngram - 1):]
171
+ for i in range(len(generated) - no_repeat_ngram + 1):
172
+ if generated[i:i + no_repeat_ngram - 1] == ngram_prefix:
173
+ blocked_token = generated[i + no_repeat_ngram - 1]
174
+ next_logits[blocked_token] = float("-inf")
175
+
176
+ next_token = next_logits.argmax().item()
177
+ generated.append(next_token)
178
+
179
+ if next_token == eos_id:
180
+ break
181
+
182
+ # Return without the initial SOS
183
+ return torch.tensor(generated[1:], dtype=torch.long)
src/repositories/__init__.py ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ from .dataset_repository import DatasetRepository
2
+ from .cache_repository import CacheRepository
src/repositories/cache_repository.py ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Cache Repository — disk and HF Hub persistence for pre-computed mel spectrograms.
3
+
4
+ Handles reading/writing individual .pt files on local disk, packing/unpacking
5
+ shards for HF Hub upload/download. Supports incremental shard uploads so
6
+ progress survives container restarts on ephemeral environments (HF Spaces).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import gc
12
+ import json
13
+ from pathlib import Path
14
+ from typing import Iterator
15
+
16
+ import torch
17
+
18
+
19
+ class CacheRepository:
20
+ """Manages a local disk cache of (mel, tokens) .pt files and
21
+ syncs with a HuggingFace Dataset repo for persistence."""
22
+
23
+ def __init__(self, cache_dir: str):
24
+ self._cache_dir = Path(cache_dir)
25
+ self._shard_dir = self._cache_dir / "_shards"
26
+
27
+ @property
28
+ def path(self) -> str:
29
+ return str(self._cache_dir)
30
+
31
+ def exists(self) -> bool:
32
+ """Check if cache has any .pt files."""
33
+ return self._cache_dir.exists() and any(self._cache_dir.glob("*.pt"))
34
+
35
+ def count(self) -> int:
36
+ """Number of cached samples."""
37
+ if not self._cache_dir.exists():
38
+ return 0
39
+ return len(list(self._cache_dir.glob("*.pt")))
40
+
41
+ def indices(self) -> list[int]:
42
+ """Sorted list of all cached sample indices."""
43
+ if not self._cache_dir.exists():
44
+ return []
45
+ return sorted(int(f.stem) for f in self._cache_dir.glob("*.pt"))
46
+
47
+ def save(self, idx: int, mel: torch.Tensor, tokens: torch.Tensor) -> None:
48
+ """Save a single (mel, tokens) pair to disk."""
49
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
50
+ torch.save((mel, tokens), self._cache_dir / f"{idx}.pt")
51
+
52
+ def load(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
53
+ """Load a single (mel, tokens) pair from disk."""
54
+ mel, tokens = torch.load(
55
+ self._cache_dir / f"{idx}.pt",
56
+ weights_only=True, map_location="cpu",
57
+ )
58
+ mel_out = mel.contiguous().clone()
59
+ tok_out = tokens.contiguous().clone()
60
+ del mel, tokens
61
+ return mel_out, tok_out
62
+
63
+ def save_stream(self, stream: Iterator[tuple[torch.Tensor, torch.Tensor]],
64
+ source_label: str = "",
65
+ repo_id: str | None = None,
66
+ samples_per_shard: int = 50000) -> int:
67
+ """Save an iterator of (mel, tokens) pairs to disk sequentially.
68
+
69
+ When repo_id is set, uploads a shard to HF Hub every samples_per_shard
70
+ samples and deletes the local .pt files. This makes caching resilient
71
+ to container restarts — progress is persisted incrementally.
72
+
73
+ Returns the number of samples saved.
74
+ """
75
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
76
+ existing = self.indices()
77
+ start_idx = (max(existing) + 1) if existing else 0
78
+ count = 0
79
+ shard_buffer: list[tuple[torch.Tensor, torch.Tensor]] = []
80
+
81
+ # Figure out next shard index from local + remote shards
82
+ shard_idx = self._next_shard_idx(repo_id=repo_id)
83
+
84
+ for mel, tokens in stream:
85
+ self.save(start_idx + count, mel, tokens)
86
+ shard_buffer.append((mel.clone(), tokens.clone()))
87
+ count += 1
88
+ del mel, tokens
89
+
90
+ if count % 1000 == 0:
91
+ label = f"{source_label}: " if source_label else ""
92
+ print(f" {label}{count} samples cached ({start_idx + count} total)...",
93
+ flush=True)
94
+ gc.collect()
95
+
96
+ # Incremental upload when shard is full
97
+ if repo_id and len(shard_buffer) >= samples_per_shard:
98
+ self._upload_shard(repo_id, shard_idx, shard_buffer)
99
+ # Delete local .pt files that were just uploaded
100
+ self._delete_range(start_idx + count - len(shard_buffer),
101
+ start_idx + count)
102
+ shard_buffer = []
103
+ shard_idx += 1
104
+ gc.collect()
105
+
106
+ # Upload remaining partial shard
107
+ if repo_id and shard_buffer:
108
+ self._upload_shard(repo_id, shard_idx, shard_buffer)
109
+ self._delete_range(start_idx + count - len(shard_buffer),
110
+ start_idx + count)
111
+ shard_buffer = []
112
+ shard_idx += 1
113
+
114
+ gc.collect()
115
+ return count
116
+
117
+ def _next_shard_idx(self, repo_id: str | None = None) -> int:
118
+ """Find the next shard index from local AND remote shard files.
119
+
120
+ Checks both local _shards/ directory and the HF Hub repo to avoid
121
+ overwriting shards that were uploaded before a container restart.
122
+ """
123
+ indices: list[int] = []
124
+
125
+ # Local shards
126
+ self._shard_dir.mkdir(parents=True, exist_ok=True)
127
+ for f in self._shard_dir.glob("shard_*.pt"):
128
+ try:
129
+ indices.append(int(f.stem.split("_")[1]))
130
+ except (IndexError, ValueError):
131
+ pass
132
+
133
+ # Remote shards (survives container restarts)
134
+ if repo_id:
135
+ try:
136
+ from huggingface_hub import list_repo_files
137
+ files = list_repo_files(repo_id, repo_type="dataset")
138
+ for fname in files:
139
+ if fname.startswith("shard_") and fname.endswith(".pt"):
140
+ try:
141
+ indices.append(int(fname.split("_")[1].split(".")[0]))
142
+ except (IndexError, ValueError):
143
+ pass
144
+ except Exception:
145
+ pass
146
+
147
+ return (max(indices) + 1) if indices else 0
148
+
149
+ def _upload_shard(self, repo_id: str, shard_idx: int,
150
+ data: list[tuple[torch.Tensor, torch.Tensor]]) -> None:
151
+ """Pack and upload a single shard to HF Hub."""
152
+ from huggingface_hub import HfApi, create_repo
153
+
154
+ api = HfApi()
155
+ try:
156
+ create_repo(repo_id, repo_type="dataset", exist_ok=True)
157
+ except Exception:
158
+ pass
159
+
160
+ self._shard_dir.mkdir(parents=True, exist_ok=True)
161
+ shard_path = self._shard_dir / f"shard_{shard_idx:04d}.pt"
162
+ torch.save(data, shard_path)
163
+ size_mb = shard_path.stat().st_size / 1e6
164
+ print(f" Uploading shard_{shard_idx:04d}: {len(data)} samples ({size_mb:.0f} MB)...",
165
+ flush=True)
166
+
167
+ api.upload_file(
168
+ path_or_fileobj=str(shard_path),
169
+ path_in_repo=f"shard_{shard_idx:04d}.pt",
170
+ repo_id=repo_id,
171
+ repo_type="dataset",
172
+ commit_message=f"Shard {shard_idx}: {len(data)} samples",
173
+ )
174
+ # Delete local shard after upload to save disk
175
+ shard_path.unlink(missing_ok=True)
176
+ print(f" Shard {shard_idx} uploaded and cleaned up.", flush=True)
177
+
178
+ def _delete_range(self, start: int, end: int) -> None:
179
+ """Delete .pt files in [start, end) to free disk after shard upload."""
180
+ for idx in range(start, end):
181
+ pt = self._cache_dir / f"{idx}.pt"
182
+ pt.unlink(missing_ok=True)
183
+
184
+ def download_from_hub(self, repo_id: str) -> int:
185
+ """Download pre-cached mel shards from a HF Dataset repo and unpack to .pt files.
186
+
187
+ Returns the total number of unpacked samples.
188
+ """
189
+ from huggingface_hub import hf_hub_download, list_repo_files
190
+
191
+ if self.exists():
192
+ n = self.count()
193
+ print(f" Found {n} cached samples in {self._cache_dir}, skipping download.")
194
+ return n
195
+
196
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
197
+ print(f" Downloading mel cache from {repo_id}...")
198
+
199
+ files = list_repo_files(repo_id, repo_type="dataset")
200
+ shard_files = sorted(f for f in files if f.startswith("shard_") and f.endswith(".pt"))
201
+ print(f" Found {len(shard_files)} shards")
202
+
203
+ sample_idx = 0
204
+ for shard_name in shard_files:
205
+ local_path = hf_hub_download(repo_id, shard_name, repo_type="dataset")
206
+ shard_data = torch.load(local_path, weights_only=True, map_location="cpu")
207
+ for mel, tokens in shard_data:
208
+ torch.save((mel, tokens), self._cache_dir / f"{sample_idx}.pt")
209
+ sample_idx += 1
210
+ del shard_data
211
+ print(f" Unpacked {shard_name} -> {sample_idx} samples total")
212
+
213
+ print(f" Downloaded and unpacked {sample_idx} samples to {self._cache_dir}")
214
+ return sample_idx
215
+
216
+ def upload_to_hub(self, repo_id: str, samples_per_shard: int = 50000) -> None:
217
+ """Pack cached .pt files into shards and upload to a HF Dataset repo."""
218
+ import json
219
+ import shutil
220
+ from huggingface_hub import HfApi, create_repo
221
+
222
+ api = HfApi()
223
+ try:
224
+ create_repo(repo_id, repo_type="dataset", exist_ok=True)
225
+ print(f" Dataset repo: https://huggingface.co/datasets/{repo_id}")
226
+ except Exception as e:
227
+ print(f" Repo creation note: {e}")
228
+
229
+ pt_files = sorted(self._cache_dir.glob("*.pt"), key=lambda f: int(f.stem))
230
+ total = len(pt_files)
231
+ print(f" Packing {total} samples into shards...")
232
+
233
+ shard_dir = self._cache_dir / "_shards"
234
+ shard_dir.mkdir(exist_ok=True)
235
+
236
+ shard_idx = 0
237
+ shard_data: list[tuple[torch.Tensor, torch.Tensor]] = []
238
+
239
+ for i, pt_file in enumerate(pt_files):
240
+ mel, tokens = torch.load(pt_file, weights_only=True, map_location="cpu")
241
+ shard_data.append((mel, tokens))
242
+
243
+ if len(shard_data) >= samples_per_shard or i == total - 1:
244
+ shard_path = shard_dir / f"shard_{shard_idx:04d}.pt"
245
+ torch.save(shard_data, shard_path)
246
+ size_mb = shard_path.stat().st_size / 1e6
247
+ print(f" Shard {shard_idx}: {len(shard_data)} samples ({size_mb:.0f} MB)")
248
+ shard_data = []
249
+ shard_idx += 1
250
+
251
+ metadata = {
252
+ "total_samples": total,
253
+ "num_shards": shard_idx,
254
+ "samples_per_shard": samples_per_shard,
255
+ }
256
+ with open(shard_dir / "metadata.json", "w") as f:
257
+ json.dump(metadata, f, indent=2)
258
+
259
+ print(f" Uploading {shard_idx} shards to {repo_id}...")
260
+ api.upload_folder(
261
+ folder_path=str(shard_dir),
262
+ repo_id=repo_id,
263
+ repo_type="dataset",
264
+ commit_message=f"Upload mel cache: {total} samples in {shard_idx} shards",
265
+ )
266
+ print(f" Upload complete!")
267
+ shutil.rmtree(shard_dir)
src/repositories/dataset_repository.py ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Dataset Repository — external data access for HuggingFace audio datasets.
3
+
4
+ Handles fetching from HF Hub, schema resolution, streaming, and
5
+ mel spectrogram extraction. Supports concurrent mel transforms via
6
+ ProcessPoolExecutor for faster caching on multi-core machines.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import os
12
+ import re
13
+ from concurrent.futures import ProcessPoolExecutor
14
+ from pathlib import Path
15
+ from typing import Iterator
16
+
17
+ import torch
18
+ import torchaudio.transforms as T
19
+ import yaml
20
+
21
+ from src.models.config import GrassmannConfig
22
+ from src.services.tokenizer_service import CharTokenizer
23
+
24
+
25
+ def _process_raw_sample(args: tuple) -> tuple[torch.Tensor, torch.Tensor] | None:
26
+ """Process a single (waveform_array, sr, text, needs_normalize) in a worker.
27
+
28
+ Top-level function so it's picklable by ProcessPoolExecutor.
29
+ Returns (mel, tokens) or None if invalid.
30
+ """
31
+ waveform_array, sr, text, needs_normalize, config_dict = args
32
+
33
+ # Reconstruct transforms in worker (not picklable across processes)
34
+ target_sr = config_dict["sample_rate"]
35
+ mel_transform = T.MelSpectrogram(
36
+ sample_rate=target_sr, n_fft=config_dict["n_fft"],
37
+ hop_length=config_dict["hop_length"], n_mels=config_dict["n_mels"],
38
+ )
39
+
40
+ try:
41
+ waveform = torch.tensor(waveform_array, dtype=torch.float32)
42
+ if sr != target_sr:
43
+ waveform = T.Resample(sr, target_sr)(waveform)
44
+ if waveform.dim() == 1:
45
+ waveform = waveform.unsqueeze(0)
46
+
47
+ mel = mel_transform(waveform)
48
+ mel = torch.log(mel.clamp(min=1e-9)).squeeze(0)
49
+ mean = mel.mean()
50
+ std = mel.std().clamp(min=1e-9)
51
+ mel = (mel - mean) / std
52
+
53
+ if needs_normalize:
54
+ text = normalize_transcript(text)
55
+
56
+ # Inline tokenization (CharTokenizer is simple enough)
57
+ vocab = {c: i + 1 for i, c in enumerate("abcdefghijklmnopqrstuvwxyz '")}
58
+ tokens = torch.tensor([vocab.get(c, 0) for c in text.lower()], dtype=torch.long)
59
+ if len(tokens) == 0:
60
+ return None
61
+
62
+ return mel, tokens
63
+ except Exception:
64
+ return None
65
+
66
+
67
+ def _load_recipes_config(config_path: str = "recipes.yaml") -> dict:
68
+ """Load dataset schemas and recipes from YAML config."""
69
+ path = Path(config_path)
70
+ if not path.exists():
71
+ return {"schemas": {}, "recipes": {}}
72
+ with open(path) as f:
73
+ return yaml.safe_load(f)
74
+
75
+
76
+ def normalize_transcript(text: str) -> str:
77
+ """Normalize transcript to lowercase ASCII with only letters, spaces, apostrophes."""
78
+ text = text.lower().strip()
79
+ text = re.sub(r"[^\w\s']", "", text)
80
+ text = re.sub(r"\s+", " ", text)
81
+ return text.strip()
82
+
83
+
84
+ class DatasetRepository:
85
+ """Fetches audio+text samples from HuggingFace Hub datasets.
86
+
87
+ Resolves column schemas automatically from recipes.yaml or known defaults.
88
+ Streams samples as (mel, tokens) pairs — the caller decides where they go
89
+ (memory, disk cache, etc.).
90
+ """
91
+
92
+ def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer,
93
+ recipes_path: str = "recipes.yaml"):
94
+ self.config = config
95
+ self.tokenizer = tokenizer
96
+ self._recipes_config = _load_recipes_config(recipes_path)
97
+ self._schemas: dict[str, dict[str, str]] = self._recipes_config.get("schemas", {})
98
+ self._recipes: dict[str, list[dict]] = self._recipes_config.get("recipes", {})
99
+ self._mel_transform = T.MelSpectrogram(
100
+ sample_rate=config.sample_rate, n_fft=config.n_fft,
101
+ hop_length=config.hop_length, n_mels=config.n_mels,
102
+ )
103
+
104
+ def resolve_columns(self, dataset_name: str,
105
+ audio_col: str | None = None,
106
+ text_col: str | None = None) -> tuple[str, str]:
107
+ """Resolve audio and text column names for a dataset."""
108
+ if audio_col and text_col:
109
+ return audio_col, text_col
110
+ for known_name, schema in self._schemas.items():
111
+ if known_name in dataset_name:
112
+ return audio_col or schema["audio"], text_col or schema["text"]
113
+ return audio_col or "audio", text_col or "text"
114
+
115
+ def get_recipe(self, name: str) -> list[dict]:
116
+ """Get a named recipe from config."""
117
+ if name not in self._recipes:
118
+ available = list(self._recipes.keys())
119
+ raise ValueError(f"Unknown recipe '{name}'. Available: {available}")
120
+ return self._recipes[name]
121
+
122
+ @property
123
+ def available_recipes(self) -> list[str]:
124
+ return list(self._recipes.keys())
125
+
126
+ def process_sample(self, sample: dict, audio_col: str,
127
+ text_col: str) -> tuple[torch.Tensor, str]:
128
+ """Extract mel spectrogram and text from a single HF sample."""
129
+ audio_data = sample[audio_col]
130
+ waveform = torch.tensor(audio_data["array"], dtype=torch.float32)
131
+ sr = audio_data["sampling_rate"]
132
+
133
+ if sr != self.config.sample_rate:
134
+ resampler = T.Resample(sr, self.config.sample_rate)
135
+ waveform = resampler(waveform)
136
+
137
+ if waveform.dim() == 1:
138
+ waveform = waveform.unsqueeze(0)
139
+
140
+ mel = self._mel_transform(waveform)
141
+ mel = torch.log(mel.clamp(min=1e-9))
142
+ mel = mel.squeeze(0)
143
+
144
+ mean = mel.mean()
145
+ std = mel.std().clamp(min=1e-9)
146
+ mel = (mel - mean) / std
147
+
148
+ return mel, sample[text_col]
149
+
150
+ def stream(self, dataset_name: str, split: str,
151
+ config_name: str | None = None,
152
+ max_samples: int | None = None,
153
+ audio_col: str | None = None,
154
+ text_col: str | None = None,
155
+ workers: int | None = None) -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
156
+ """Stream (mel, tokens) pairs from a HF dataset.
157
+
158
+ Uses ProcessPoolExecutor to compute mel spectrograms concurrently.
159
+ Workers defaults to CPU count. Set workers=0 for sequential mode.
160
+ """
161
+ from datasets import load_dataset
162
+
163
+ a_col, t_col = self.resolve_columns(dataset_name, audio_col, text_col)
164
+ needs_normalize = "librispeech" not in dataset_name.lower()
165
+ n_workers = workers if workers is not None else min(os.cpu_count() or 1, 8)
166
+
167
+ ds = load_dataset(dataset_name, config_name, split=split, streaming=True)
168
+ count = 0
169
+
170
+ # Config dict for worker processes (can't pickle MelSpectrogram)
171
+ config_dict = {
172
+ "sample_rate": self.config.sample_rate,
173
+ "n_fft": self.config.n_fft,
174
+ "hop_length": self.config.hop_length,
175
+ "n_mels": self.config.n_mels,
176
+ }
177
+
178
+ # Sequential fallback
179
+ if n_workers < 2:
180
+ yield from self._stream_sequential(
181
+ ds, a_col, t_col, needs_normalize, max_samples)
182
+ return
183
+
184
+ # Concurrent: prefetch batches of raw samples, fan out mel transforms
185
+ batch_size = n_workers * 4 # keep workers fed
186
+ raw_batch: list[tuple] = []
187
+
188
+ for sample in ds:
189
+ if max_samples and count >= max_samples:
190
+ break
191
+
192
+ try:
193
+ raw_text = sample[t_col].strip()
194
+ except (KeyError, AttributeError):
195
+ continue
196
+
197
+ text = normalize_transcript(raw_text) if needs_normalize else raw_text
198
+ if len(text) == 0:
199
+ continue
200
+
201
+ try:
202
+ audio_array = sample[a_col]["array"]
203
+ except (KeyError, TypeError):
204
+ continue
205
+
206
+ if len(audio_array) < 1600 or len(audio_array) > 240000:
207
+ continue
208
+
209
+ sr = sample[a_col]["sampling_rate"]
210
+ raw_batch.append((audio_array, sr, text, needs_normalize, config_dict))
211
+
212
+ if len(raw_batch) >= batch_size:
213
+ for result in self._process_batch(raw_batch, n_workers):
214
+ if max_samples and count >= max_samples:
215
+ break
216
+ yield result
217
+ count += 1
218
+ raw_batch = []
219
+
220
+ # Drain remaining batch
221
+ if raw_batch and (not max_samples or count < max_samples):
222
+ for result in self._process_batch(raw_batch, n_workers):
223
+ if max_samples and count >= max_samples:
224
+ break
225
+ yield result
226
+ count += 1
227
+
228
+ del ds
229
+
230
+ def _process_batch(self, raw_batch: list[tuple],
231
+ n_workers: int) -> list[tuple[torch.Tensor, torch.Tensor]]:
232
+ """Process a batch of raw samples concurrently."""
233
+ results = []
234
+ with ProcessPoolExecutor(max_workers=n_workers) as pool:
235
+ for result in pool.map(_process_raw_sample, raw_batch):
236
+ if result is not None:
237
+ results.append(result)
238
+ return results
239
+
240
+ def _stream_sequential(self, ds, a_col: str, t_col: str,
241
+ needs_normalize: bool,
242
+ max_samples: int | None) -> Iterator[tuple[torch.Tensor, torch.Tensor]]:
243
+ """Sequential fallback for stream()."""
244
+ count = 0
245
+ for sample in ds:
246
+ if max_samples and count >= max_samples:
247
+ break
248
+
249
+ try:
250
+ raw_text = sample[t_col].strip()
251
+ except (KeyError, AttributeError):
252
+ continue
253
+
254
+ text = normalize_transcript(raw_text) if needs_normalize else raw_text
255
+ if len(text) == 0:
256
+ continue
257
+
258
+ try:
259
+ audio_array = sample[a_col]["array"]
260
+ except (KeyError, TypeError):
261
+ continue
262
+
263
+ if len(audio_array) < 1600 or len(audio_array) > 240000:
264
+ continue
265
+
266
+ try:
267
+ mel, _ = self.process_sample(sample, a_col, t_col)
268
+ except Exception:
269
+ continue
270
+
271
+ if needs_normalize:
272
+ text = normalize_transcript(raw_text)
273
+ tokens = self.tokenizer.encode(text)
274
+ if len(tokens) == 0:
275
+ continue
276
+
277
+ yield mel, tokens
278
+ count += 1
279
+
280
+ del ds
281
+
282
+ def fetch_full(self, dataset_name: str, split: str,
283
+ config_name: str | None = None):
284
+ """Download a full (non-streaming) HF dataset split. Returns the HF dataset object."""
285
+ from datasets import load_dataset
286
+ return load_dataset(dataset_name, config_name, split=split)
src/services/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from .audio_service import AudioService
2
+ from .tokenizer_service import CharTokenizer
3
+ from .training_service import TrainingService
src/services/audio_service.py ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torchaudio
3
+ import torchaudio.transforms as T
4
+
5
+ from src.interfaces import IAudioProcessor
6
+ from src.models.config import GrassmannConfig
7
+
8
+
9
+ class AudioService(IAudioProcessor):
10
+ """Handles audio loading and mel spectrogram extraction."""
11
+
12
+ def __init__(self, config: GrassmannConfig):
13
+ self.config = config
14
+ self.mel_transform = T.MelSpectrogram(
15
+ sample_rate=config.sample_rate,
16
+ n_fft=config.n_fft,
17
+ hop_length=config.hop_length,
18
+ n_mels=config.n_mels,
19
+ )
20
+
21
+ def load(self, path: str) -> torch.Tensor:
22
+ waveform, sr = torchaudio.load(path)
23
+ # Resample if needed
24
+ if sr != self.config.sample_rate:
25
+ resampler = T.Resample(sr, self.config.sample_rate)
26
+ waveform = resampler(waveform)
27
+ # Convert to mono
28
+ if waveform.shape[0] > 1:
29
+ waveform = waveform.mean(dim=0, keepdim=True)
30
+ return waveform.squeeze(0)
31
+
32
+ def extract_features(self, waveform: torch.Tensor) -> torch.Tensor:
33
+ if waveform.dim() == 1:
34
+ waveform = waveform.unsqueeze(0)
35
+ mel = self.mel_transform(waveform) # (1, n_mels, time)
36
+ # Log mel
37
+ mel = torch.log(mel.clamp(min=1e-9))
38
+ return mel.squeeze(0) # (n_mels, time)
39
+
40
+ def normalize(self, features: torch.Tensor) -> torch.Tensor:
41
+ mean = features.mean()
42
+ std = features.std().clamp(min=1e-9)
43
+ return (features - mean) / std
src/services/dataset_service.py ADDED
@@ -0,0 +1,295 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Dataset Service — orchestrates data loading, splitting, and DataLoader creation.
3
+
4
+ Delegates external data access to repositories:
5
+ - DatasetRepository: HF Hub fetching, schema resolution, streaming
6
+ - CacheRepository: disk cache read/write, HF shard upload/download
7
+
8
+ This service handles: train/val/test splitting, PyTorch Dataset wrappers,
9
+ DataLoader creation, and collation.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from typing import Self
15
+
16
+ import torch
17
+ import torchaudio.transforms as T
18
+ from torch.nn.utils.rnn import pad_sequence
19
+ from torch.utils.data import DataLoader, Dataset
20
+
21
+ from src.models.config import GrassmannConfig
22
+ from src.repositories.cache_repository import CacheRepository
23
+ from src.repositories.dataset_repository import DatasetRepository
24
+ from src.services.tokenizer_service import CharTokenizer
25
+
26
+
27
+ # -- PyTorch Dataset wrappers ---------------------------------------------
28
+
29
+ class MaterializedAudioDataset(Dataset):
30
+ """Pre-materialized dataset -- samples already in memory as (mel, tokens) pairs."""
31
+
32
+ def __init__(self, samples: list[tuple[torch.Tensor, torch.Tensor]]):
33
+ self.samples = samples
34
+
35
+ def __len__(self) -> int:
36
+ return len(self.samples)
37
+
38
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
39
+ return self.samples[idx]
40
+
41
+
42
+ class DiskCachedDataset(Dataset):
43
+ """Dataset backed by individual .pt files on disk."""
44
+
45
+ def __init__(self, cache: CacheRepository, indices: list[int]):
46
+ self._cache = cache
47
+ self._indices = indices
48
+
49
+ def __len__(self) -> int:
50
+ return len(self._indices)
51
+
52
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
53
+ return self._cache.load(self._indices[idx])
54
+
55
+
56
+ class HFAudioDataset(Dataset):
57
+ """Wraps a (non-streaming) HuggingFace dataset split into a PyTorch Dataset."""
58
+
59
+ def __init__(self, hf_dataset, repo: DatasetRepository,
60
+ audio_col: str, text_col: str):
61
+ self.hf_dataset = hf_dataset
62
+ self._repo = repo
63
+ self.audio_col = audio_col
64
+ self.text_col = text_col
65
+
66
+ def __len__(self) -> int:
67
+ return len(self.hf_dataset)
68
+
69
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
70
+ sample = self.hf_dataset[idx]
71
+ mel, text = self._repo.process_sample(sample, self.audio_col, self.text_col)
72
+ tokens = self._repo.tokenizer.encode(text)
73
+ return mel, tokens
74
+
75
+
76
+ def collate_fn(
77
+ batch: list[tuple[torch.Tensor, torch.Tensor]],
78
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
79
+ """Pad mels and tokens to batch-max lengths."""
80
+ mels, tokens = zip(*batch)
81
+ mel_lengths = torch.tensor([m.shape[-1] for m in mels], dtype=torch.long)
82
+ max_mel_len = mel_lengths.max().item()
83
+ padded_mels = []
84
+ for m in mels:
85
+ pad_amount = max_mel_len - m.shape[-1]
86
+ padded_mels.append(torch.nn.functional.pad(m, (0, pad_amount)))
87
+ mel_batch = torch.stack(padded_mels)
88
+ token_lengths = torch.tensor([len(t) for t in tokens], dtype=torch.long)
89
+ token_batch = pad_sequence(tokens, batch_first=True, padding_value=0)
90
+ return mel_batch, token_batch, mel_lengths, token_lengths
91
+
92
+
93
+ # -- Service ---------------------------------------------------------------
94
+
95
+ class DatasetService:
96
+ """Orchestrates data loading, splitting, and DataLoader creation.
97
+
98
+ Usage:
99
+ repo = DatasetRepository(config, tokenizer)
100
+ cache = CacheRepository("checkpoints/mel_cache")
101
+ svc = DatasetService(repo, cache)
102
+
103
+ # From HF Hub cache:
104
+ svc.load_from_hub_cache("tisaconundrum/grassmann-mel-cache")
105
+ svc.split(max_samples=28539)
106
+
107
+ # From recipe:
108
+ svc.load_recipe("1750h")
109
+ svc.split()
110
+
111
+ # Direct HF download:
112
+ svc.load_dataset("openslr/librispeech_asr", split="train.clean.100")
113
+ svc.split()
114
+
115
+ train_loader = svc.get_train_loader(batch_size=32)
116
+ """
117
+
118
+ def __init__(self, repo: DatasetRepository, cache: CacheRepository):
119
+ self._repo = repo
120
+ self._cache = cache
121
+ self._hf_train = None
122
+ self._hf_val = None
123
+ self._hf_test = None
124
+ self._materialized: list[tuple[torch.Tensor, torch.Tensor]] | None = None
125
+ self._use_cache = False
126
+ self._audio_col = "audio"
127
+ self._text_col = "text"
128
+ self._train_ds: Dataset | None = None
129
+ self._val_ds: Dataset | None = None
130
+ self._test_ds: Dataset | None = None
131
+
132
+ def load_from_hub_cache(self, repo_id: str) -> Self:
133
+ """Download pre-cached mel shards from a HF Dataset repo."""
134
+ self._cache.download_from_hub(repo_id)
135
+ self._use_cache = True
136
+ return self
137
+
138
+ def load_recipe(self, recipe_name: str,
139
+ max_samples_per_source: int | None = None,
140
+ upload_repo: str | None = None,
141
+ samples_per_shard: int = 50000,
142
+ workers: int = 0) -> Self:
143
+ """Stream multiple datasets per recipe and cache to disk.
144
+
145
+ When upload_repo is set, shards are uploaded incrementally to HF Hub
146
+ as they fill up — progress survives container restarts.
147
+ """
148
+ sources = self._repo.get_recipe(recipe_name)
149
+ print(f" Loading recipe '{recipe_name}': {len(sources)} sources")
150
+ if upload_repo:
151
+ print(f" Incremental upload to: {upload_repo}")
152
+ for src in sources:
153
+ cfg_str = f" ({src['config']})" if src.get('config') else ""
154
+ print(f" - {src['name']} / {src['split']}{cfg_str}")
155
+
156
+ for src in sources:
157
+ src_cap = src.get("max_samples") or max_samples_per_source
158
+ stream = self._repo.stream(
159
+ src["name"], src["split"],
160
+ config_name=src.get("config"),
161
+ max_samples=src_cap,
162
+ workers=workers,
163
+ )
164
+ label = f"{src['name']}/{src['split']}"
165
+ count = self._cache.save_stream(
166
+ stream, source_label=label,
167
+ repo_id=upload_repo, samples_per_shard=samples_per_shard,
168
+ )
169
+ print(f" {label}: {count} samples")
170
+
171
+ self._use_cache = True
172
+ return self
173
+
174
+ def load_dataset(self, dataset_name: str, split: str | None = None,
175
+ config_name: str | None = None,
176
+ max_samples: int | None = None) -> Self:
177
+ """Load a single dataset -- streaming or full download."""
178
+ self._audio_col, self._text_col = self._repo.resolve_columns(dataset_name)
179
+ print(f" Loading dataset: {dataset_name}")
180
+ print(f" Columns: audio='{self._audio_col}', text='{self._text_col}'")
181
+
182
+ if max_samples is not None:
183
+ stream = self._repo.stream(
184
+ dataset_name, split or "train",
185
+ config_name=config_name, max_samples=max_samples,
186
+ )
187
+ self._materialized = list(stream)
188
+ print(f" Materialized {len(self._materialized)} samples into memory.")
189
+ return self
190
+
191
+ # Full download
192
+ if split:
193
+ self._hf_train = self._repo.fetch_full(dataset_name, split, config_name)
194
+ print(f" Loaded split '{split}': {len(self._hf_train)} samples")
195
+ else:
196
+ try:
197
+ self._hf_train = self._repo.fetch_full(dataset_name, "train", config_name)
198
+ except Exception:
199
+ pass
200
+ try:
201
+ self._hf_val = self._repo.fetch_full(dataset_name, "validation", config_name)
202
+ except Exception:
203
+ pass
204
+ try:
205
+ self._hf_test = self._repo.fetch_full(dataset_name, "test", config_name)
206
+ except Exception:
207
+ pass
208
+ sizes = []
209
+ if self._hf_train: sizes.append(f"train={len(self._hf_train)}")
210
+ if self._hf_val: sizes.append(f"val={len(self._hf_val)}")
211
+ if self._hf_test: sizes.append(f"test={len(self._hf_test)}")
212
+ print(f" Loaded splits: {', '.join(sizes)}")
213
+ return self
214
+
215
+ def clean(self) -> Self:
216
+ """Filter out empty/invalid samples from full-download datasets."""
217
+ if self._hf_train is not None and hasattr(self._hf_train, "filter"):
218
+ original_len = len(self._hf_train)
219
+ a, t = self._audio_col, self._text_col
220
+ self._hf_train = self._hf_train.filter(
221
+ lambda x: len(x[t].strip()) > 0 and len(x[a]["array"]) > 1600
222
+ )
223
+ print(f" Cleaned: {original_len} -> {len(self._hf_train)} samples")
224
+ return self
225
+
226
+ def preprocess(self) -> Self:
227
+ return self
228
+
229
+ def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1,
230
+ max_samples: int | None = None) -> Self:
231
+ """Split loaded data into train/val/test."""
232
+ if self._use_cache:
233
+ all_indices = self._cache.indices()
234
+ if max_samples is not None:
235
+ all_indices = all_indices[:max_samples]
236
+ n = len(all_indices)
237
+ n_train = max(1, int(n * train_ratio))
238
+ n_val = max(1, int(n * val_ratio))
239
+ self._train_ds = DiskCachedDataset(self._cache, all_indices[:n_train])
240
+ self._val_ds = DiskCachedDataset(self._cache, all_indices[n_train:n_train + n_val])
241
+ rest = all_indices[n_train + n_val:]
242
+ self._test_ds = DiskCachedDataset(self._cache, rest if rest else all_indices[n_train:n_train + n_val])
243
+
244
+ elif self._materialized is not None:
245
+ samples = self._materialized
246
+ n = len(samples)
247
+ n_train = max(1, int(n * train_ratio))
248
+ n_val = max(1, int(n * val_ratio))
249
+ self._train_ds = MaterializedAudioDataset(samples[:n_train])
250
+ self._val_ds = MaterializedAudioDataset(samples[n_train:n_train + n_val])
251
+ rest = samples[n_train + n_val:]
252
+ self._test_ds = MaterializedAudioDataset(rest if rest else samples[n_train:n_train + n_val])
253
+
254
+ elif self._hf_val is not None:
255
+ self._train_ds = HFAudioDataset(self._hf_train, self._repo, self._audio_col, self._text_col)
256
+ self._val_ds = HFAudioDataset(self._hf_val, self._repo, self._audio_col, self._text_col)
257
+ self._test_ds = HFAudioDataset(self._hf_test, self._repo, self._audio_col, self._text_col) if self._hf_test else self._val_ds
258
+
259
+ elif self._hf_train is not None:
260
+ n = len(self._hf_train)
261
+ n_train = int(n * train_ratio)
262
+ n_val = int(n * val_ratio)
263
+ split_ds = self._hf_train.train_test_split(test_size=(n - n_train) / n, seed=42)
264
+ train_hf = split_ds["train"]
265
+ remaining = split_ds["test"]
266
+ if len(remaining) > 1:
267
+ val_test = remaining.train_test_split(
268
+ test_size=max(1, n - n_train - n_val) / len(remaining), seed=42)
269
+ val_hf, test_hf = val_test["train"], val_test["test"]
270
+ else:
271
+ val_hf = test_hf = remaining
272
+ self._train_ds = HFAudioDataset(train_hf, self._repo, self._audio_col, self._text_col)
273
+ self._val_ds = HFAudioDataset(val_hf, self._repo, self._audio_col, self._text_col)
274
+ self._test_ds = HFAudioDataset(test_hf, self._repo, self._audio_col, self._text_col)
275
+
276
+ if self._train_ds:
277
+ print(f" Splits ready: train={len(self._train_ds)}, "
278
+ f"val={len(self._val_ds) if self._val_ds else 0}, "
279
+ f"test={len(self._test_ds) if self._test_ds else 0}")
280
+ return self
281
+
282
+ def get_train_loader(self, batch_size: int) -> DataLoader:
283
+ assert self._train_ds is not None, "Call load_*() and split() first"
284
+ return DataLoader(self._train_ds, batch_size=batch_size, shuffle=True,
285
+ collate_fn=collate_fn, num_workers=0)
286
+
287
+ def get_val_loader(self, batch_size: int) -> DataLoader:
288
+ assert self._val_ds is not None, "Call load_*() and split() first"
289
+ return DataLoader(self._val_ds, batch_size=batch_size, shuffle=False,
290
+ collate_fn=collate_fn, num_workers=0)
291
+
292
+ def get_test_loader(self, batch_size: int) -> DataLoader:
293
+ assert self._test_ds is not None, "Call load_*() and split() first"
294
+ return DataLoader(self._test_ds, batch_size=batch_size, shuffle=False,
295
+ collate_fn=collate_fn, num_workers=0)
src/services/eval_service.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Evaluation Service.
3
+
4
+ WER/CER evaluation on LibriSpeech (or other HF datasets).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import time
10
+
11
+ import torch
12
+ import torchaudio
13
+ import torchaudio.transforms
14
+
15
+ from src.models.config import GrassmannConfig
16
+ from src.services.tokenizer_service import CharTokenizer
17
+
18
+
19
+ _mel_transform = None
20
+
21
+
22
+ def make_mel(waveform: torch.Tensor, sample_rate: int, config: GrassmannConfig) -> torch.Tensor:
23
+ """Convert waveform to normalized mel spectrogram (matches training pipeline)."""
24
+ global _mel_transform
25
+ if _mel_transform is None:
26
+ _mel_transform = torchaudio.transforms.MelSpectrogram(
27
+ sample_rate=config.sample_rate, n_fft=config.n_fft,
28
+ hop_length=config.hop_length, n_mels=config.n_mels,
29
+ )
30
+
31
+ if sample_rate != config.sample_rate:
32
+ waveform = torchaudio.functional.resample(waveform, sample_rate, config.sample_rate)
33
+ if waveform.dim() == 1:
34
+ waveform = waveform.unsqueeze(0)
35
+
36
+ mel = _mel_transform(waveform)
37
+ mel = torch.log(mel.clamp(min=1e-9)).squeeze(0)
38
+
39
+ mean = mel.mean()
40
+ std = mel.std().clamp(min=1e-9)
41
+ mel = (mel - mean) / std
42
+
43
+ return mel.unsqueeze(0)
44
+
45
+
46
+ @torch.no_grad()
47
+ def greedy_decode(model, mel: torch.Tensor, tokenizer: CharTokenizer) -> str:
48
+ log_probs = model(mel)
49
+ preds = log_probs.argmax(dim=-1).squeeze(1)
50
+ return tokenizer.decode(preds)
51
+
52
+
53
+ def evaluate_wer(
54
+ model,
55
+ config: GrassmannConfig,
56
+ tokenizer: CharTokenizer,
57
+ split: str = "test.clean",
58
+ max_samples: int | None = None,
59
+ show_errors: int = 10,
60
+ ) -> dict:
61
+ """Run WER/CER evaluation on a LibriSpeech split.
62
+
63
+ Returns dict with wer, cer, num_samples, elapsed_seconds.
64
+ """
65
+ from jiwer import wer as compute_wer, cer as compute_cer
66
+ from datasets import load_dataset
67
+
68
+ model.eval()
69
+ print(f" Loading LibriSpeech {split}...")
70
+ ds = load_dataset("openslr/librispeech_asr", split=split, streaming=True)
71
+
72
+ references, hypotheses, errors = [], [], []
73
+ start = time.time()
74
+
75
+ for i, sample in enumerate(ds):
76
+ if max_samples and i >= max_samples:
77
+ break
78
+
79
+ audio = sample["audio"]
80
+ waveform = torch.tensor(audio["array"], dtype=torch.float32).unsqueeze(0)
81
+ sr = audio["sampling_rate"]
82
+ ref = sample["text"].lower().strip()
83
+
84
+ mel = make_mel(waveform, sr, config)
85
+ hyp = greedy_decode(model, mel, tokenizer).strip()
86
+
87
+ references.append(ref)
88
+ hypotheses.append(hyp)
89
+
90
+ sample_wer = compute_wer(ref, hyp) if ref else 0.0
91
+ errors.append((sample_wer, ref, hyp))
92
+
93
+ if (i + 1) % 100 == 0:
94
+ elapsed = time.time() - start
95
+ current_wer = compute_wer(references, hypotheses)
96
+ print(f" {i+1} samples | WER: {current_wer:.2%} | {elapsed:.0f}s", flush=True)
97
+
98
+ total_time = time.time() - start
99
+ total_wer = compute_wer(references, hypotheses)
100
+ total_cer = compute_cer(references, hypotheses)
101
+
102
+ print(f"\n{'=' * 60}")
103
+ print(f" LibriSpeech {split} — Greedy CTC Decode")
104
+ print(f"{'=' * 60}")
105
+ print(f" Samples: {len(references)}")
106
+ print(f" WER: {total_wer:.2%}")
107
+ print(f" CER: {total_cer:.2%}")
108
+ print(f" Time: {total_time:.1f}s ({total_time/len(references):.2f}s/sample)")
109
+ print(f"{'=' * 60}")
110
+
111
+ if show_errors > 0:
112
+ errors.sort(key=lambda x: -x[0])
113
+ print(f"\n Top {show_errors} worst transcriptions:")
114
+ for j, (w, ref, hyp) in enumerate(errors[:show_errors]):
115
+ print(f"\n [{j+1}] WER: {w:.0%}")
116
+ print(f" GT: '{ref}'")
117
+ print(f" PRED: '{hyp}'")
118
+
119
+ return {
120
+ "wer": total_wer,
121
+ "cer": total_cer,
122
+ "num_samples": len(references),
123
+ "elapsed_seconds": total_time,
124
+ }
src/services/export_service.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ ONNX Export Service.
3
+
4
+ Handles exporting Grassmann models to ONNX format for cross-platform inference.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+
11
+ import torch
12
+
13
+ from src.models.config import GrassmannConfig
14
+
15
+
16
+ class EncoderWrapper(torch.nn.Module):
17
+ def __init__(self, encoder):
18
+ super().__init__()
19
+ self.encoder = encoder
20
+
21
+ def forward(self, mel: torch.Tensor) -> torch.Tensor:
22
+ return self.encoder(mel)
23
+
24
+
25
+ class DecoderStepWrapper(torch.nn.Module):
26
+ def __init__(self, decoder):
27
+ super().__init__()
28
+ self.decoder = decoder
29
+
30
+ def forward(self, tokens: torch.Tensor, encoder_out: torch.Tensor) -> torch.Tensor:
31
+ logits = self.decoder(tokens, encoder_out)
32
+ return logits[:, -1:, :]
33
+
34
+
35
+ class CTCModelWrapper(torch.nn.Module):
36
+ def __init__(self, model):
37
+ super().__init__()
38
+ self.frontend = model.frontend
39
+ self.pos_enc = model.pos_enc
40
+ self.layers = model.layers
41
+ self.output_proj = model.output_proj
42
+
43
+ def forward(self, mel: torch.Tensor) -> torch.Tensor:
44
+ x = self.frontend(mel)
45
+ x = self.pos_enc(x)
46
+ for layer in self.layers:
47
+ x = layer(x)
48
+ logits = self.output_proj(x)
49
+ return logits.log_softmax(dim=-1)
50
+
51
+
52
+ def export_seq2seq(model, config: GrassmannConfig, output_dir: str, opset: int = 18) -> None:
53
+ model = model.cpu()
54
+ model.eval()
55
+ out = Path(output_dir)
56
+ out.mkdir(parents=True, exist_ok=True)
57
+
58
+ print(" Exporting encoder.onnx...")
59
+ encoder_wrapper = EncoderWrapper(model.encoder)
60
+ encoder_wrapper.eval()
61
+ dummy_mel = torch.randn(1, config.n_mels, 200)
62
+ torch.onnx.export(
63
+ encoder_wrapper, (dummy_mel,), str(out / "encoder.onnx"),
64
+ input_names=["mel"], output_names=["encoder_out"],
65
+ dynamic_axes={"mel": {0: "batch", 2: "time"}, "encoder_out": {0: "batch", 1: "enc_time"}},
66
+ opset_version=opset,
67
+ )
68
+
69
+ print(" Exporting decoder_step.onnx...")
70
+ decoder_wrapper = DecoderStepWrapper(model.decoder)
71
+ decoder_wrapper.eval()
72
+ dummy_tokens = torch.tensor([[0, 5, 12]], dtype=torch.long)
73
+ dummy_enc_out = torch.randn(1, 50, config.d_model)
74
+ torch.onnx.export(
75
+ decoder_wrapper, (dummy_tokens, dummy_enc_out), str(out / "decoder_step.onnx"),
76
+ input_names=["tokens", "encoder_out"], output_names=["next_logits"],
77
+ dynamic_axes={"tokens": {0: "batch", 1: "seq_len"}, "encoder_out": {0: "batch", 1: "enc_time"}, "next_logits": {0: "batch"}},
78
+ opset_version=opset,
79
+ )
80
+
81
+
82
+ def export_ctc(model, config: GrassmannConfig, output_dir: str, opset: int = 18) -> None:
83
+ model = model.cpu()
84
+ model.eval()
85
+ out = Path(output_dir)
86
+ out.mkdir(parents=True, exist_ok=True)
87
+
88
+ print(" Exporting model.onnx (CTC)...")
89
+ wrapper = CTCModelWrapper(model)
90
+ wrapper.eval()
91
+ dummy_mel = torch.randn(1, config.n_mels, 200)
92
+ torch.onnx.export(
93
+ wrapper, (dummy_mel,), str(out / "model.onnx"),
94
+ input_names=["mel"], output_names=["log_probs"],
95
+ dynamic_axes={"mel": {0: "batch", 2: "time"}, "log_probs": {0: "batch", 1: "time"}},
96
+ opset_version=opset,
97
+ )
98
+
99
+
100
+ def verify_onnx(output_dir: str, config: GrassmannConfig, is_seq2seq: bool) -> None:
101
+ try:
102
+ import onnxruntime as ort
103
+ except ImportError:
104
+ print(" Install onnxruntime to verify: pip install onnxruntime")
105
+ return
106
+
107
+ import numpy as np
108
+ out = Path(output_dir)
109
+
110
+ if is_seq2seq:
111
+ print(" Verifying encoder.onnx...")
112
+ enc_session = ort.InferenceSession(str(out / "encoder.onnx"))
113
+ mel = np.random.randn(1, config.n_mels, 200).astype(np.float32)
114
+ enc_out = enc_session.run(None, {"mel": mel})[0]
115
+ print(f" Encoder output shape: {enc_out.shape}")
116
+
117
+ print(" Verifying decoder_step.onnx...")
118
+ dec_session = ort.InferenceSession(str(out / "decoder_step.onnx"))
119
+ tokens = np.array([[0, 5, 12]], dtype=np.int64)
120
+ logits = dec_session.run(None, {"tokens": tokens, "encoder_out": enc_out})[0]
121
+ print(f" Decoder logits shape: {logits.shape}")
122
+ else:
123
+ print(" Verifying model.onnx (CTC)...")
124
+ session = ort.InferenceSession(str(out / "model.onnx"))
125
+ mel = np.random.randn(1, config.n_mels, 200).astype(np.float32)
126
+ log_probs = session.run(None, {"mel": mel})[0]
127
+ print(f" Output shape: {log_probs.shape}")
128
+
129
+ print(" ONNX verification passed!")
src/services/hf_dataset_service.py ADDED
@@ -0,0 +1,657 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ HuggingFace Dataset Service.
3
+
4
+ Loads any HuggingFace audio+text dataset (LibriSpeech, Common Voice, FLEURS, etc.)
5
+ and wraps it into PyTorch DataLoaders for the Grassmann transcription pipeline.
6
+
7
+ Uses streaming mode by default when max_samples is set, so we only download
8
+ the rows we actually need instead of multi-GB parquet shards.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import Self
14
+
15
+ import torch
16
+ import torchaudio.transforms as T
17
+ from torch.nn.utils.rnn import pad_sequence
18
+ from torch.utils.data import DataLoader, Dataset
19
+
20
+ from src.interfaces import IDataset
21
+ from src.models.config import GrassmannConfig
22
+ from src.services.tokenizer_service import CharTokenizer
23
+
24
+
25
+ import re
26
+
27
+ KNOWN_SCHEMAS: dict[str, tuple[str, str]] = {
28
+ "openslr/librispeech_asr": ("audio", "text"),
29
+ "mozilla-foundation/common_voice": ("audio", "sentence"),
30
+ "google/fleurs": ("audio", "transcription"),
31
+ "speechcolab/gigaspeech": ("audio", "text"),
32
+ "facebook/voxpopuli": ("audio", "normalized_text"),
33
+ "MLCommons/peoples_speech": ("audio", "text"),
34
+ "espnet/yodas": ("audio", "text"),
35
+ }
36
+
37
+ # Predefined multi-dataset recipes
38
+ DATASET_RECIPES: dict[str, list[dict]] = {
39
+ "librispeech-960": [
40
+ {"name": "openslr/librispeech_asr", "split": "train.clean.100"},
41
+ {"name": "openslr/librispeech_asr", "split": "train.clean.360"},
42
+ {"name": "openslr/librispeech_asr", "split": "train.other.500"},
43
+ ],
44
+ "1750h": [
45
+ {"name": "openslr/librispeech_asr", "split": "train.clean.100"},
46
+ {"name": "openslr/librispeech_asr", "split": "train.clean.360"},
47
+ {"name": "openslr/librispeech_asr", "split": "train.other.500"},
48
+ {"name": "facebook/voxpopuli", "config": "en", "split": "train"},
49
+ {"name": "MLCommons/peoples_speech", "config": "clean", "split": "train",
50
+ "max_samples": 30000}, # ~250h subset of 30K hours
51
+ ],
52
+ }
53
+
54
+
55
+ def normalize_transcript(text: str) -> str:
56
+ """Normalize transcript to lowercase ASCII with only letters, spaces, apostrophes."""
57
+ text = text.lower().strip()
58
+ text = re.sub(r"[^\w\s']", "", text)
59
+ text = re.sub(r"\s+", " ", text)
60
+ return text.strip()
61
+
62
+
63
+ def _resolve_columns(dataset_name: str, audio_col: str | None, text_col: str | None) -> tuple[str, str]:
64
+ if audio_col and text_col:
65
+ return audio_col, text_col
66
+ for known_name, (a, t) in KNOWN_SCHEMAS.items():
67
+ if known_name in dataset_name:
68
+ return audio_col or a, text_col or t
69
+ return audio_col or "audio", text_col or "text"
70
+
71
+
72
+ def _process_sample(
73
+ sample: dict,
74
+ audio_col: str,
75
+ text_col: str,
76
+ mel_transform: T.MelSpectrogram,
77
+ target_sr: int,
78
+ ) -> tuple[torch.Tensor, str]:
79
+ """Extract mel spectrogram and text from a single HF sample."""
80
+ audio_data = sample[audio_col]
81
+ waveform = torch.tensor(audio_data["array"], dtype=torch.float32)
82
+ sr = audio_data["sampling_rate"]
83
+
84
+ if sr != target_sr:
85
+ resampler = T.Resample(sr, target_sr)
86
+ waveform = resampler(waveform)
87
+
88
+ if waveform.dim() == 1:
89
+ waveform = waveform.unsqueeze(0)
90
+
91
+ mel = mel_transform(waveform)
92
+ mel = torch.log(mel.clamp(min=1e-9))
93
+ mel = mel.squeeze(0)
94
+
95
+ mean = mel.mean()
96
+ std = mel.std().clamp(min=1e-9)
97
+ mel = (mel - mean) / std
98
+
99
+ return mel, sample[text_col]
100
+
101
+
102
+ class MaterializedAudioDataset(Dataset):
103
+ """Pre-materialized dataset -- samples already in memory as (mel, tokens) pairs."""
104
+
105
+ def __init__(self, samples: list[tuple[torch.Tensor, torch.Tensor]]):
106
+ self.samples = samples
107
+
108
+ def __len__(self) -> int:
109
+ return len(self.samples)
110
+
111
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
112
+ return self.samples[idx]
113
+
114
+
115
+ class DiskCachedDataset(Dataset):
116
+ """Dataset backed by individual .pt files on disk. Low RAM usage.
117
+
118
+ Uses a single index file to avoid thousands of small file opens.
119
+ On first access, builds an in-memory index mapping idx -> (file_offset).
120
+ """
121
+
122
+ def __init__(self, cache_dir: str, indices: list[int] | None = None):
123
+ from pathlib import Path
124
+ self._cache_dir = Path(cache_dir)
125
+ # If indices provided, use subset; otherwise use all files
126
+ if indices is not None:
127
+ self._indices = indices
128
+ else:
129
+ self._indices = sorted(
130
+ int(f.stem) for f in self._cache_dir.glob("*.pt")
131
+ )
132
+
133
+ def __len__(self) -> int:
134
+ return len(self._indices)
135
+
136
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
137
+ file_idx = self._indices[idx]
138
+ mel, tokens = torch.load(
139
+ self._cache_dir / f"{file_idx}.pt", weights_only=True,
140
+ map_location="cpu",
141
+ )
142
+ # Clone to detach from the file-backed storage, then delete originals
143
+ mel_out, tok_out = mel.contiguous().clone(), tokens.contiguous().clone()
144
+ del mel, tokens
145
+ return mel_out, tok_out
146
+
147
+
148
+ class HFAudioDataset(Dataset):
149
+ """Wraps a (non-streaming) HuggingFace dataset split into a PyTorch Dataset."""
150
+
151
+ def __init__(self, hf_dataset, audio_col: str, text_col: str,
152
+ tokenizer: CharTokenizer, config: GrassmannConfig):
153
+ self.hf_dataset = hf_dataset
154
+ self.audio_col = audio_col
155
+ self.text_col = text_col
156
+ self.tokenizer = tokenizer
157
+ self.mel_transform = T.MelSpectrogram(
158
+ sample_rate=config.sample_rate, n_fft=config.n_fft,
159
+ hop_length=config.hop_length, n_mels=config.n_mels,
160
+ )
161
+ self.target_sr = config.sample_rate
162
+
163
+ def __len__(self) -> int:
164
+ return len(self.hf_dataset)
165
+
166
+ def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]:
167
+ sample = self.hf_dataset[idx]
168
+ mel, text = _process_sample(
169
+ sample, self.audio_col, self.text_col, self.mel_transform, self.target_sr
170
+ )
171
+ tokens = self.tokenizer.encode(text)
172
+ return mel, tokens
173
+
174
+
175
+ def hf_collate_fn(
176
+ batch: list[tuple[torch.Tensor, torch.Tensor]],
177
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
178
+ mels, tokens = zip(*batch)
179
+ mel_lengths = torch.tensor([m.shape[-1] for m in mels], dtype=torch.long)
180
+ max_mel_len = mel_lengths.max().item()
181
+ padded_mels = []
182
+ for m in mels:
183
+ pad_amount = max_mel_len - m.shape[-1]
184
+ padded_mels.append(torch.nn.functional.pad(m, (0, pad_amount)))
185
+ mel_batch = torch.stack(padded_mels)
186
+ token_lengths = torch.tensor([len(t) for t in tokens], dtype=torch.long)
187
+ token_batch = pad_sequence(tokens, batch_first=True, padding_value=0)
188
+ return mel_batch, token_batch, mel_lengths, token_lengths
189
+
190
+
191
+ def _stream_and_materialize(
192
+ dataset_name: str,
193
+ config_name: str | None,
194
+ split: str,
195
+ max_samples: int,
196
+ audio_col: str,
197
+ text_col: str,
198
+ tokenizer: CharTokenizer,
199
+ config: GrassmannConfig,
200
+ cache_dir: str | None = None,
201
+ ) -> list[tuple[torch.Tensor, torch.Tensor]] | str:
202
+ """Stream a HF dataset and save processed samples.
203
+
204
+ If cache_dir is set, saves each (mel, tokens) pair as a .pt file on disk
205
+ and returns the cache_dir path (low RAM). Otherwise materializes into a list
206
+ in memory (original behavior).
207
+ """
208
+ from pathlib import Path
209
+ from datasets import load_dataset
210
+
211
+ # Check if cache already exists
212
+ if cache_dir:
213
+ cache_path = Path(cache_dir)
214
+ existing = list(cache_path.glob("*.pt"))
215
+ if existing:
216
+ print(f" Found {len(existing)} cached samples in {cache_dir}, skipping stream.")
217
+ return cache_dir
218
+ cache_path.mkdir(parents=True, exist_ok=True)
219
+
220
+ print(f" Streaming {max_samples} samples from '{split}'...")
221
+ ds = load_dataset(dataset_name, config_name, split=split, streaming=True)
222
+
223
+ mel_transform = T.MelSpectrogram(
224
+ sample_rate=config.sample_rate, n_fft=config.n_fft,
225
+ hop_length=config.hop_length, n_mels=config.n_mels,
226
+ )
227
+
228
+ import gc
229
+
230
+ samples: list[tuple[torch.Tensor, torch.Tensor]] = []
231
+ saved_count = 0
232
+ for i, sample in enumerate(ds):
233
+ if i >= max_samples:
234
+ break
235
+ text = sample[text_col].strip()
236
+ audio_array = sample[audio_col]["array"]
237
+ # Skip empty, too-short (<0.1s), or too-long (>15s at 16kHz = 240000 samples) clips
238
+ # Long clips cause massive padding waste in collate_fn
239
+ if len(text) == 0 or len(audio_array) < 1600 or len(audio_array) > 240000:
240
+ del sample, audio_array
241
+ continue
242
+ mel, text = _process_sample(sample, audio_col, text_col, mel_transform, config.sample_rate)
243
+ tokens = tokenizer.encode(text)
244
+ if cache_dir:
245
+ torch.save((mel, tokens), Path(cache_dir) / f"{saved_count}.pt")
246
+ saved_count += 1
247
+ del mel, tokens
248
+ else:
249
+ samples.append((mel, tokens))
250
+ del sample, audio_array
251
+ if (i + 1) % 10 == 0:
252
+ print(f" Processed {i + 1}/{max_samples} samples...")
253
+ # Periodic GC during caching to prevent RAM buildup from librosa/datasets buffers
254
+ if cache_dir and (i + 1) % 500 == 0:
255
+ gc.collect()
256
+
257
+ # Explicitly release the streaming dataset and its internal buffers
258
+ del ds
259
+ gc.collect()
260
+
261
+ if cache_dir:
262
+ print(f" Cached {saved_count} samples to disk at {cache_dir}.")
263
+ return cache_dir
264
+ else:
265
+ print(f" Materialized {len(samples)} samples into memory.")
266
+ return samples
267
+
268
+
269
+ def _stream_multi_datasets(
270
+ sources: list[dict],
271
+ tokenizer: CharTokenizer,
272
+ config: GrassmannConfig,
273
+ cache_dir: str,
274
+ max_samples_per_source: int | None = None,
275
+ ) -> str:
276
+ """Stream multiple HF datasets and cache all samples to disk.
277
+
278
+ Each source dict has: name, split, and optionally config.
279
+ Non-LibriSpeech datasets get normalize_transcript() applied.
280
+
281
+ Returns the cache_dir path.
282
+ """
283
+ from pathlib import Path
284
+ from datasets import load_dataset
285
+ import gc
286
+
287
+ cache_path = Path(cache_dir)
288
+ existing = list(cache_path.glob("*.pt"))
289
+ if existing:
290
+ print(f" Found {len(existing)} cached samples in {cache_dir}, skipping stream.")
291
+ return cache_dir
292
+ cache_path.mkdir(parents=True, exist_ok=True)
293
+
294
+ mel_transform = T.MelSpectrogram(
295
+ sample_rate=config.sample_rate, n_fft=config.n_fft,
296
+ hop_length=config.hop_length, n_mels=config.n_mels,
297
+ )
298
+
299
+ saved_count = 0
300
+ for src in sources:
301
+ name = src["name"]
302
+ split = src["split"]
303
+ src_config = src.get("config")
304
+ audio_col, text_col = _resolve_columns(name, None, None)
305
+ needs_normalize = "librispeech" not in name.lower()
306
+
307
+ config_str = f" ({src_config})" if src_config else ""
308
+ norm_str = " (normalizing)" if needs_normalize else ""
309
+ print(f"\n [{saved_count} cached] Streaming {name} / {split}{config_str}{norm_str}...")
310
+
311
+ ds = load_dataset(name, src_config, split=split, streaming=True)
312
+ src_count = 0
313
+ # Per-source cap: use source-level override, then global cap, then unlimited
314
+ src_cap = src.get("max_samples") or max_samples_per_source
315
+
316
+ for i, sample in enumerate(ds):
317
+ if src_cap and src_count >= src_cap:
318
+ break
319
+
320
+ try:
321
+ raw_text = sample[text_col].strip()
322
+ except (KeyError, AttributeError):
323
+ continue
324
+
325
+ text = normalize_transcript(raw_text) if needs_normalize else raw_text
326
+ if len(text) == 0:
327
+ continue
328
+
329
+ try:
330
+ audio_array = sample[audio_col]["array"]
331
+ except (KeyError, TypeError):
332
+ continue
333
+
334
+ # Skip too-short (<0.1s) or too-long (>15s) clips
335
+ if len(audio_array) < 1600 or len(audio_array) > 240000:
336
+ continue
337
+
338
+ try:
339
+ mel, _ = _process_sample(sample, audio_col, text_col, mel_transform, config.sample_rate)
340
+ except Exception:
341
+ continue
342
+
343
+ # Re-normalize the text (process_sample returns raw text)
344
+ if needs_normalize:
345
+ text = normalize_transcript(raw_text)
346
+ tokens = tokenizer.encode(text)
347
+
348
+ if len(tokens) == 0:
349
+ continue
350
+
351
+ torch.save((mel, tokens), cache_path / f"{saved_count}.pt")
352
+ saved_count += 1
353
+ src_count += 1
354
+
355
+ del mel, tokens, sample, audio_array
356
+ if src_count % 1000 == 0:
357
+ print(f" {name}: {src_count} samples cached ({saved_count} total)...")
358
+ gc.collect()
359
+
360
+ del ds
361
+ gc.collect()
362
+ print(f" {name}/{split}: {src_count} samples")
363
+
364
+ print(f"\n Total: {saved_count} samples cached to {cache_dir}")
365
+ return cache_dir
366
+
367
+
368
+ class HFDatasetService(IDataset):
369
+ """Loads audio datasets from HuggingFace Hub.
370
+
371
+ Usage:
372
+ _dataset = HFDatasetService(config, tokenizer)
373
+ _dataset.load("openslr/librispeech_asr", split="train.clean.100",
374
+ max_samples=100)
375
+ .clean().preprocess().split()
376
+
377
+ # Multi-dataset mode:
378
+ _dataset.load_multi("1750h", cache_dir="checkpoints/mel_cache")
379
+ .clean().preprocess().split()
380
+ """
381
+
382
+ def __init__(self, config: GrassmannConfig, tokenizer: CharTokenizer):
383
+ self.config = config
384
+ self.tokenizer = tokenizer
385
+ self.dataset_name: str = ""
386
+ self.audio_col: str = "audio"
387
+ self.text_col: str = "text"
388
+ self._hf_train = None
389
+ self._hf_val = None
390
+ self._hf_test = None
391
+ self._materialized: list[tuple[torch.Tensor, torch.Tensor]] | None = None
392
+ self._cache_dir: str | None = None
393
+ self._train_ds: Dataset | None = None
394
+ self._val_ds: Dataset | None = None
395
+ self._test_ds: Dataset | None = None
396
+
397
+ def load(self, dataset_name: str, config_name: str | None = None,
398
+ split: str | None = None, audio_col: str | None = None,
399
+ text_col: str | None = None, streaming: bool = False,
400
+ max_samples: int | None = None,
401
+ cache_dir: str | None = None) -> Self:
402
+ from datasets import load_dataset
403
+
404
+ self.dataset_name = dataset_name
405
+ self.audio_col, self.text_col = _resolve_columns(dataset_name, audio_col, text_col)
406
+ print(f" Loading dataset: {dataset_name}")
407
+ print(f" Columns: audio='{self.audio_col}', text='{self.text_col}'")
408
+
409
+ # When max_samples is set, stream to avoid downloading everything
410
+ if max_samples is not None:
411
+ train_split = split or "train"
412
+ result = _stream_and_materialize(
413
+ dataset_name, config_name, train_split, max_samples,
414
+ self.audio_col, self.text_col, self.tokenizer, self.config,
415
+ cache_dir=cache_dir,
416
+ )
417
+ if isinstance(result, str):
418
+ self._cache_dir = result
419
+ else:
420
+ self._materialized = result
421
+ return self
422
+
423
+ # Full download mode
424
+ if split:
425
+ ds = load_dataset(dataset_name, config_name, split=split)
426
+ self._hf_train = ds
427
+ print(f" Loaded split '{split}': {len(ds)} samples")
428
+ else:
429
+ try:
430
+ self._hf_train = load_dataset(dataset_name, config_name, split="train")
431
+ except Exception:
432
+ pass
433
+ try:
434
+ self._hf_val = load_dataset(dataset_name, config_name, split="validation")
435
+ except Exception:
436
+ pass
437
+ try:
438
+ self._hf_test = load_dataset(dataset_name, config_name, split="test")
439
+ except Exception:
440
+ pass
441
+ sizes = []
442
+ if self._hf_train: sizes.append(f"train={len(self._hf_train)}")
443
+ if self._hf_val: sizes.append(f"val={len(self._hf_val)}")
444
+ if self._hf_test: sizes.append(f"test={len(self._hf_test)}")
445
+ print(f" Loaded splits: {', '.join(sizes)}")
446
+ return self
447
+
448
+ def load_multi(
449
+ self,
450
+ recipe: str | list[dict],
451
+ cache_dir: str,
452
+ max_samples_per_source: int | None = None,
453
+ ) -> Self:
454
+ """Load multiple datasets using a recipe name or explicit source list.
455
+
456
+ Args:
457
+ recipe: Either a key from DATASET_RECIPES (e.g. "1750h", "librispeech-960")
458
+ or a list of dicts with name/split/config keys.
459
+ cache_dir: Directory to cache processed mel+token pairs.
460
+ max_samples_per_source: Cap samples per dataset (for testing).
461
+ """
462
+ if isinstance(recipe, str):
463
+ if recipe not in DATASET_RECIPES:
464
+ raise ValueError(f"Unknown recipe '{recipe}'. Available: {list(DATASET_RECIPES.keys())}")
465
+ sources = DATASET_RECIPES[recipe]
466
+ print(f" Loading recipe '{recipe}': {len(sources)} sources")
467
+ else:
468
+ sources = recipe
469
+ print(f" Loading {len(sources)} custom sources")
470
+
471
+ for src in sources:
472
+ cfg_str = f" ({src['config']})" if src.get('config') else ""
473
+ print(f" - {src['name']} / {src['split']}{cfg_str}")
474
+
475
+ self._cache_dir = _stream_multi_datasets(
476
+ sources, self.tokenizer, self.config, cache_dir,
477
+ max_samples_per_source=max_samples_per_source,
478
+ )
479
+ return self
480
+
481
+ def load_from_hub_cache(self, repo_id: str, cache_dir: str) -> Self:
482
+ """Download pre-cached mel shards from a HF Dataset repo and unpack to .pt files.
483
+
484
+ Use this to skip expensive CPU caching when mels have already been
485
+ pre-computed and uploaded via upload_cache().
486
+ """
487
+ from pathlib import Path
488
+ from huggingface_hub import hf_hub_download, list_repo_files
489
+
490
+ cache_path = Path(cache_dir)
491
+ existing = list(cache_path.glob("*.pt"))
492
+ if existing:
493
+ print(f" Found {len(existing)} cached samples in {cache_dir}, skipping download.")
494
+ self._cache_dir = cache_dir
495
+ return self
496
+
497
+ cache_path.mkdir(parents=True, exist_ok=True)
498
+ print(f" Downloading mel cache from {repo_id}...")
499
+
500
+ files = list_repo_files(repo_id, repo_type="dataset")
501
+ shard_files = sorted(f for f in files if f.startswith("shard_") and f.endswith(".pt"))
502
+ print(f" Found {len(shard_files)} shards")
503
+
504
+ sample_idx = 0
505
+ for shard_name in shard_files:
506
+ local_path = hf_hub_download(repo_id, shard_name, repo_type="dataset")
507
+ shard_data = torch.load(local_path, weights_only=True, map_location="cpu")
508
+ for mel, tokens in shard_data:
509
+ torch.save((mel, tokens), cache_path / f"{sample_idx}.pt")
510
+ sample_idx += 1
511
+ del shard_data
512
+ print(f" Unpacked {shard_name} -> {sample_idx} samples total")
513
+
514
+ print(f" Downloaded and unpacked {sample_idx} samples to {cache_dir}")
515
+ self._cache_dir = cache_dir
516
+ return self
517
+
518
+ def upload_cache(self, repo_id: str) -> None:
519
+ """Upload the mel cache directory as shards to a HF Dataset repo.
520
+
521
+ Call after load_multi() to persist the cache so GPU training
522
+ can use load_from_hub_cache() instead of re-streaming.
523
+ """
524
+ import json
525
+ import shutil
526
+ from pathlib import Path
527
+ from huggingface_hub import HfApi, create_repo
528
+
529
+ assert self._cache_dir is not None, "No cache to upload. Call load_multi() first."
530
+
531
+ api = HfApi()
532
+ try:
533
+ create_repo(repo_id, repo_type="dataset", exist_ok=True)
534
+ print(f" Dataset repo: https://huggingface.co/datasets/{repo_id}")
535
+ except Exception as e:
536
+ print(f" Repo creation note: {e}")
537
+
538
+ cache_path = Path(self._cache_dir)
539
+ pt_files = sorted(cache_path.glob("*.pt"), key=lambda f: int(f.stem))
540
+ total = len(pt_files)
541
+ print(f" Packing {total} samples into shards...")
542
+
543
+ shard_dir = cache_path / "_shards"
544
+ shard_dir.mkdir(exist_ok=True)
545
+
546
+ SAMPLES_PER_SHARD = 50000
547
+ shard_idx = 0
548
+ shard_data = []
549
+
550
+ for i, pt_file in enumerate(pt_files):
551
+ mel, tokens = torch.load(pt_file, weights_only=True, map_location="cpu")
552
+ shard_data.append((mel, tokens))
553
+
554
+ if len(shard_data) >= SAMPLES_PER_SHARD or i == total - 1:
555
+ shard_path = shard_dir / f"shard_{shard_idx:04d}.pt"
556
+ torch.save(shard_data, shard_path)
557
+ size_mb = shard_path.stat().st_size / 1e6
558
+ print(f" Shard {shard_idx}: {len(shard_data)} samples ({size_mb:.0f} MB)")
559
+ shard_data = []
560
+ shard_idx += 1
561
+
562
+ metadata = {
563
+ "total_samples": total,
564
+ "num_shards": shard_idx,
565
+ "samples_per_shard": SAMPLES_PER_SHARD,
566
+ }
567
+ with open(shard_dir / "metadata.json", "w") as f:
568
+ json.dump(metadata, f, indent=2)
569
+
570
+ print(f" Uploading {shard_idx} shards to {repo_id}...")
571
+ api.upload_folder(
572
+ folder_path=str(shard_dir),
573
+ repo_id=repo_id,
574
+ repo_type="dataset",
575
+ commit_message=f"Upload mel cache: {total} samples in {shard_idx} shards",
576
+ )
577
+ print(f" Upload complete!")
578
+ shutil.rmtree(shard_dir)
579
+
580
+ def clean(self) -> Self:
581
+ if self._materialized is not None:
582
+ return self # already filtered during streaming
583
+ if self._hf_train is not None and hasattr(self._hf_train, "filter"):
584
+ original_len = len(self._hf_train)
585
+ self._hf_train = self._hf_train.filter(
586
+ lambda x: len(x[self.text_col].strip()) > 0
587
+ and len(x[self.audio_col]["array"]) > 1600
588
+ )
589
+ print(f" Cleaned: {original_len} -> {len(self._hf_train)} samples")
590
+ return self
591
+
592
+ def preprocess(self) -> Self:
593
+ return self
594
+
595
+ def split(self, train_ratio: float = 0.8, val_ratio: float = 0.1,
596
+ max_samples: int | None = None) -> Self:
597
+ if self._cache_dir is not None:
598
+ # Disk-cached mode: split by file indices
599
+ from pathlib import Path
600
+ all_indices = sorted(
601
+ int(f.stem) for f in Path(self._cache_dir).glob("*.pt")
602
+ )
603
+ if max_samples is not None:
604
+ all_indices = all_indices[:max_samples]
605
+ n = len(all_indices)
606
+ n_train = max(1, int(n * train_ratio))
607
+ n_val = max(1, int(n * val_ratio))
608
+ self._train_ds = DiskCachedDataset(self._cache_dir, all_indices[:n_train])
609
+ self._val_ds = DiskCachedDataset(self._cache_dir, all_indices[n_train:n_train + n_val])
610
+ rest = all_indices[n_train + n_val:]
611
+ self._test_ds = DiskCachedDataset(self._cache_dir, rest if rest else all_indices[n_train:n_train + n_val])
612
+ elif self._materialized is not None:
613
+ samples = self._materialized
614
+ n = len(samples)
615
+ n_train = max(1, int(n * train_ratio))
616
+ n_val = max(1, int(n * val_ratio))
617
+ self._train_ds = MaterializedAudioDataset(samples[:n_train])
618
+ self._val_ds = MaterializedAudioDataset(samples[n_train:n_train + n_val])
619
+ rest = samples[n_train + n_val:]
620
+ self._test_ds = MaterializedAudioDataset(rest if rest else samples[n_train:n_train + n_val])
621
+ elif self._hf_val is not None:
622
+ self._train_ds = HFAudioDataset(self._hf_train, self.audio_col, self.text_col, self.tokenizer, self.config)
623
+ self._val_ds = HFAudioDataset(self._hf_val, self.audio_col, self.text_col, self.tokenizer, self.config)
624
+ self._test_ds = HFAudioDataset(self._hf_test, self.audio_col, self.text_col, self.tokenizer, self.config) if self._hf_test else self._val_ds
625
+ elif self._hf_train is not None:
626
+ n = len(self._hf_train)
627
+ n_train = int(n * train_ratio)
628
+ n_val = int(n * val_ratio)
629
+ split_ds = self._hf_train.train_test_split(test_size=(n - n_train) / n, seed=42)
630
+ train_hf = split_ds["train"]
631
+ remaining = split_ds["test"]
632
+ if len(remaining) > 1:
633
+ val_test = remaining.train_test_split(test_size=max(1, n - n_train - n_val) / len(remaining), seed=42)
634
+ val_hf, test_hf = val_test["train"], val_test["test"]
635
+ else:
636
+ val_hf = test_hf = remaining
637
+ self._train_ds = HFAudioDataset(train_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
638
+ self._val_ds = HFAudioDataset(val_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
639
+ self._test_ds = HFAudioDataset(test_hf, self.audio_col, self.text_col, self.tokenizer, self.config)
640
+
641
+ if self._train_ds:
642
+ print(f" Splits ready: train={len(self._train_ds)}, "
643
+ f"val={len(self._val_ds) if self._val_ds else 0}, "
644
+ f"test={len(self._test_ds) if self._test_ds else 0}")
645
+ return self
646
+
647
+ def get_train_loader(self, batch_size: int) -> DataLoader:
648
+ assert self._train_ds is not None, "Call load() and split() first"
649
+ return DataLoader(self._train_ds, batch_size=batch_size, shuffle=True, collate_fn=hf_collate_fn, num_workers=0)
650
+
651
+ def get_val_loader(self, batch_size: int) -> DataLoader:
652
+ assert self._val_ds is not None, "Call load() and split() first"
653
+ return DataLoader(self._val_ds, batch_size=batch_size, shuffle=False, collate_fn=hf_collate_fn, num_workers=0)
654
+
655
+ def get_test_loader(self, batch_size: int) -> DataLoader:
656
+ assert self._test_ds is not None, "Call load() and split() first"
657
+ return DataLoader(self._test_ds, batch_size=batch_size, shuffle=False, collate_fn=hf_collate_fn, num_workers=0)
src/services/model_hub_service.py ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Model Hub Service — Save/load models in a self-contained format.
3
+
4
+ Bundles config + weights + tokenizer info into a single directory
5
+ that anyone can download and run inference with.
6
+
7
+ Saved format:
8
+ model_dir/
9
+ config.json # GrassmannConfig as JSON
10
+ model.pt # state_dict weights
11
+ tokenizer_info.json # vocab info for reconstruction
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from dataclasses import asdict
18
+ from pathlib import Path
19
+
20
+ import torch
21
+
22
+ from src.models.config import GrassmannConfig
23
+ from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
24
+ from src.models.grassmann_model import GrassmannTranscriptionModel
25
+ from src.services.tokenizer_service import CharTokenizer
26
+
27
+
28
+ def save_model(
29
+ model: GrassmannSeq2SeqModel | GrassmannTranscriptionModel,
30
+ config: GrassmannConfig,
31
+ save_dir: str,
32
+ ) -> None:
33
+ """Save model + config in a self-contained directory."""
34
+ path = Path(save_dir)
35
+ path.mkdir(parents=True, exist_ok=True)
36
+
37
+ # Save config
38
+ config_dict = asdict(config)
39
+ with open(path / "config.json", "w") as f:
40
+ json.dump(config_dict, f, indent=2)
41
+
42
+ # Save weights
43
+ torch.save(model.state_dict(), path / "model.pt")
44
+
45
+ # Save tokenizer info
46
+ tokenizer = CharTokenizer()
47
+ tok_info = {
48
+ "type": "char",
49
+ "vocab_size": tokenizer.vocab_size,
50
+ "blank_id": tokenizer.blank_id,
51
+ }
52
+ with open(path / "tokenizer_info.json", "w") as f:
53
+ json.dump(tok_info, f, indent=2)
54
+
55
+ print(f" Model saved to {save_dir}/")
56
+ print(f" config.json, model.pt, tokenizer_info.json")
57
+
58
+
59
+ def load_model(
60
+ load_dir: str,
61
+ device: str = "cpu",
62
+ ) -> tuple[GrassmannSeq2SeqModel | GrassmannTranscriptionModel, GrassmannConfig, CharTokenizer]:
63
+ """Load a saved model from directory. Returns (model, config, tokenizer)."""
64
+ path = Path(load_dir)
65
+
66
+ # Load config
67
+ with open(path / "config.json") as f:
68
+ config_dict = json.load(f)
69
+ config = GrassmannConfig(**config_dict)
70
+
71
+ # Build model
72
+ tokenizer = CharTokenizer()
73
+ config.vocab_size = tokenizer.vocab_size
74
+
75
+ if config.use_seq2seq:
76
+ model = GrassmannSeq2SeqModel(config)
77
+ else:
78
+ model = GrassmannTranscriptionModel(config)
79
+
80
+ # Load weights
81
+ state = torch.load(path / "model.pt", map_location=device, weights_only=True)
82
+ model.load_state_dict(state)
83
+ model.to(device)
84
+ model.eval()
85
+
86
+ return model, config, tokenizer
src/services/seq2seq_training_service.py ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Seq2Seq Training Service.
3
+
4
+ Handles teacher-forced training for the Grassmann encoder-decoder model.
5
+ Uses cross-entropy loss instead of CTC, with proper token shifting
6
+ and padding masks.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import dataclass
12
+
13
+ import torch
14
+ import torch.nn as nn
15
+ from torch.utils.data import DataLoader
16
+
17
+ from src.models.config import GrassmannConfig
18
+ from src.models.grassmann_seq2seq import GrassmannSeq2SeqModel
19
+
20
+
21
+ @dataclass
22
+ class TrainMetrics:
23
+ epoch: int
24
+ train_loss: float
25
+ val_loss: float | None = None
26
+
27
+
28
+ class Seq2SeqTrainingService:
29
+ """Training loop for the fully attention-free encoder-decoder."""
30
+
31
+ def __init__(
32
+ self,
33
+ model: GrassmannSeq2SeqModel,
34
+ config: GrassmannConfig,
35
+ device: torch.device | None = None,
36
+ ):
37
+ self.model = model
38
+ self.config = config
39
+
40
+ if device is not None:
41
+ self.device = device
42
+ elif torch.cuda.is_available():
43
+ self.device = torch.device("cuda")
44
+ else:
45
+ self.device = torch.device("cpu")
46
+
47
+ self.model.to(self.device)
48
+
49
+ self.optimizer = torch.optim.AdamW(
50
+ model.parameters(),
51
+ lr=config.learning_rate,
52
+ weight_decay=config.weight_decay,
53
+ )
54
+ self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
55
+
56
+ # Cross-entropy loss, ignoring padding (index 0)
57
+ self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
58
+ self.history: list[TrainMetrics] = []
59
+
60
+ def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
61
+ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
62
+ warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
63
+ cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
64
+ self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
65
+
66
+ def train_epoch(self, train_loader: DataLoader) -> float:
67
+ self.model.train()
68
+ total_loss = 0.0
69
+ n_batches = 0
70
+ total_batches = len(train_loader)
71
+
72
+ for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
73
+ mels = mels.to(self.device)
74
+ tokens = tokens.to(self.device)
75
+
76
+ self.optimizer.zero_grad()
77
+
78
+ # Teacher-forced forward pass
79
+ logits = self.model(mels, tokens) # (B, T_dec, vocab)
80
+
81
+ # Reshape for cross-entropy: (B*T, vocab) vs (B*T,)
82
+ B, T, V = logits.shape
83
+ loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
84
+
85
+ if torch.isfinite(loss):
86
+ loss.backward()
87
+ torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
88
+ self.optimizer.step()
89
+ total_loss += loss.item()
90
+ n_batches += 1
91
+
92
+ # Log every 10 batches for CI visibility
93
+ if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
94
+ avg = total_loss / max(n_batches, 1)
95
+ print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
96
+
97
+ return total_loss / max(n_batches, 1)
98
+
99
+ @torch.no_grad()
100
+ def validate(self, val_loader: DataLoader) -> float:
101
+ self.model.eval()
102
+ total_loss = 0.0
103
+ n_batches = 0
104
+
105
+ for mels, tokens, mel_lengths, token_lengths in val_loader:
106
+ mels = mels.to(self.device)
107
+ tokens = tokens.to(self.device)
108
+
109
+ logits = self.model(mels, tokens)
110
+
111
+ B, T, V = logits.shape
112
+ loss = self.criterion(logits.reshape(B * T, V), tokens.reshape(B * T))
113
+
114
+ if torch.isfinite(loss):
115
+ total_loss += loss.item()
116
+ n_batches += 1
117
+
118
+ return total_loss / max(n_batches, 1)
119
+
120
+ def train_loop(
121
+ self,
122
+ train_loader: DataLoader,
123
+ val_loader: DataLoader | None,
124
+ epochs: int,
125
+ callback: object | None = None,
126
+ ) -> list[TrainMetrics]:
127
+ for epoch in range(1, epochs + 1):
128
+ train_loss = self.train_epoch(train_loader)
129
+ val_loss = self.validate(val_loader) if val_loader else None
130
+
131
+ metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
132
+ self.history.append(metrics)
133
+
134
+ if callback and hasattr(callback, "on_epoch_end"):
135
+ callback.on_epoch_end(metrics)
136
+
137
+ if self.scheduler is not None:
138
+ self.scheduler.step()
139
+
140
+ return self.history
141
+
142
+ @torch.no_grad()
143
+ def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
144
+ """Autoregressive greedy decoding for a single mel spectrogram."""
145
+ self.model.eval()
146
+ if mel.dim() == 2:
147
+ mel = mel.unsqueeze(0)
148
+ mel = mel.to(self.device)
149
+ return self.model.greedy_decode(mel, max_len=200)
src/services/tokenizer_service.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ import torch
6
+
7
+ from src.interfaces import ITokenizer
8
+
9
+
10
+ class CharTokenizer(ITokenizer):
11
+ """
12
+ Character-level tokenizer for transcription.
13
+
14
+ Vocabulary: <blank> + space + a-z + apostrophe
15
+ """
16
+
17
+ def __init__(self):
18
+ self._chars = " " + "abcdefghijklmnopqrstuvwxyz" + "'"
19
+ self._blank = "<blank>"
20
+ self._char_to_idx: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
21
+ self._idx_to_char: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
22
+ # Index 0 is reserved for CTC blank
23
+ self._idx_to_char[0] = ""
24
+
25
+ def encode(self, text: str) -> torch.Tensor:
26
+ text = text.lower().strip()
27
+ indices = []
28
+ for c in text:
29
+ if c in self._char_to_idx:
30
+ indices.append(self._char_to_idx[c])
31
+ return torch.tensor(indices, dtype=torch.long)
32
+
33
+ def decode(self, tokens: torch.Tensor) -> str:
34
+ chars = []
35
+ prev = -1
36
+ for idx in tokens.tolist():
37
+ if idx == self.blank_id:
38
+ prev = idx
39
+ continue
40
+ if idx != prev:
41
+ chars.append(self._idx_to_char.get(idx, ""))
42
+ prev = idx
43
+ return "".join(chars)
44
+
45
+ @property
46
+ def vocab_size(self) -> int:
47
+ return len(self._chars) + 1 # +1 for blank
48
+
49
+ @property
50
+ def blank_id(self) -> int:
51
+ return 0
52
+
53
+
54
+ class BPETokenizer(ITokenizer):
55
+ """
56
+ BPE (Byte Pair Encoding) tokenizer for transcription.
57
+
58
+ Uses a simple word-level BPE trained on the fly from training transcripts.
59
+ Fewer output steps = better CTC alignment.
60
+ Token 0 is reserved for CTC blank.
61
+ """
62
+
63
+ def __init__(self, vocab_size: int = 1000):
64
+ self._target_vocab_size = vocab_size
65
+ # Start with character-level vocab
66
+ self._chars = list(" abcdefghijklmnopqrstuvwxyz'")
67
+ # Token 0 = blank, tokens 1..N = actual symbols
68
+ self._token_to_id: dict[str, int] = {c: i + 1 for i, c in enumerate(self._chars)}
69
+ self._id_to_token: dict[int, str] = {i + 1: c for i, c in enumerate(self._chars)}
70
+ self._id_to_token[0] = ""
71
+ self._trained = False
72
+
73
+ def train_from_texts(self, texts: list[str], max_merges: int | None = None) -> None:
74
+ """Train BPE merges from a list of transcripts."""
75
+ max_merges = max_merges or (self._target_vocab_size - len(self._chars) - 1)
76
+
77
+ # Tokenize all texts into characters
78
+ corpus: list[list[str]] = []
79
+ for text in texts:
80
+ text = text.lower().strip()
81
+ chars = [c for c in text if c in self._token_to_id]
82
+ if chars:
83
+ corpus.append(chars)
84
+
85
+ # BPE merge loop
86
+ merges: list[tuple[str, str]] = []
87
+ for _ in range(max_merges):
88
+ # Count adjacent pairs
89
+ pair_counts: dict[tuple[str, str], int] = {}
90
+ for tokens in corpus:
91
+ for i in range(len(tokens) - 1):
92
+ pair = (tokens[i], tokens[i + 1])
93
+ pair_counts[pair] = pair_counts.get(pair, 0) + 1
94
+
95
+ if not pair_counts:
96
+ break
97
+
98
+ # Find most frequent pair
99
+ best_pair = max(pair_counts, key=pair_counts.get) # type: ignore
100
+ merged = best_pair[0] + best_pair[1]
101
+ merges.append(best_pair)
102
+
103
+ # Add to vocab
104
+ new_id = len(self._id_to_token)
105
+ self._token_to_id[merged] = new_id
106
+ self._id_to_token[new_id] = merged
107
+
108
+ # Apply merge to corpus
109
+ for idx, tokens in enumerate(corpus):
110
+ new_tokens = []
111
+ i = 0
112
+ while i < len(tokens):
113
+ if i < len(tokens) - 1 and (tokens[i], tokens[i + 1]) == best_pair:
114
+ new_tokens.append(merged)
115
+ i += 2
116
+ else:
117
+ new_tokens.append(tokens[i])
118
+ i += 1
119
+ corpus[idx] = new_tokens
120
+
121
+ self._merges = merges
122
+ self._trained = True
123
+ print(f" BPE trained: {len(merges)} merges, vocab size {self.vocab_size}", flush=True)
124
+
125
+ def encode(self, text: str) -> torch.Tensor:
126
+ text = text.lower().strip()
127
+ tokens = [c for c in text if c in self._token_to_id]
128
+
129
+ # Apply merges in order
130
+ if self._trained:
131
+ for a, b in self._merges:
132
+ merged = a + b
133
+ new_tokens = []
134
+ i = 0
135
+ while i < len(tokens):
136
+ if i < len(tokens) - 1 and tokens[i] == a and tokens[i + 1] == b:
137
+ new_tokens.append(merged)
138
+ i += 2
139
+ else:
140
+ new_tokens.append(tokens[i])
141
+ i += 1
142
+ tokens = new_tokens
143
+
144
+ indices = [self._token_to_id[t] for t in tokens if t in self._token_to_id]
145
+ return torch.tensor(indices, dtype=torch.long)
146
+
147
+ def decode(self, tokens: torch.Tensor) -> str:
148
+ pieces = []
149
+ prev = -1
150
+ for idx in tokens.tolist():
151
+ if idx == self.blank_id:
152
+ prev = idx
153
+ continue
154
+ if idx != prev:
155
+ pieces.append(self._id_to_token.get(idx, ""))
156
+ prev = idx
157
+ return "".join(pieces)
158
+
159
+ @property
160
+ def vocab_size(self) -> int:
161
+ return len(self._id_to_token)
162
+
163
+ @property
164
+ def blank_id(self) -> int:
165
+ return 0
166
+
167
+
168
+ def create_tokenizer(config_type: str = "char", **kwargs) -> ITokenizer:
169
+ """Factory function to create the right tokenizer."""
170
+ if config_type == "bpe":
171
+ return BPETokenizer(vocab_size=kwargs.get("vocab_size", 1000))
172
+ return CharTokenizer()
src/services/training_service.py ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+ from torch.utils.data import DataLoader
8
+
9
+ from src.models.config import GrassmannConfig
10
+ from src.models.grassmann_model import GrassmannTranscriptionModel
11
+
12
+
13
+ @dataclass
14
+ class TrainMetrics:
15
+ epoch: int
16
+ train_loss: float
17
+ val_loss: float | None = None
18
+
19
+
20
+ class TrainingService:
21
+ """Handles the training loop, loss computation, and optimization."""
22
+
23
+ def __init__(
24
+ self,
25
+ model: GrassmannTranscriptionModel,
26
+ config: GrassmannConfig,
27
+ device: torch.device | None = None,
28
+ ):
29
+ self.model = model
30
+ self.config = config
31
+ if device is not None:
32
+ self.device = device
33
+ elif torch.cuda.is_available():
34
+ self.device = torch.device("cuda")
35
+ else:
36
+ # MPS can be unstable with some PyTorch/Python combos; prefer CPU as fallback
37
+ self.device = torch.device("cpu")
38
+ self.model.to(self.device)
39
+
40
+ self.optimizer = torch.optim.AdamW(
41
+ model.parameters(),
42
+ lr=config.learning_rate,
43
+ weight_decay=config.weight_decay,
44
+ )
45
+ self.scheduler: torch.optim.lr_scheduler.LRScheduler | None = None
46
+ self.ctc_loss = nn.CTCLoss(blank=0, zero_infinity=True)
47
+ self.history: list[TrainMetrics] = []
48
+
49
+ def set_cosine_schedule(self, epochs: int, warmup_epochs: int = 2) -> None:
50
+ """Add cosine annealing LR schedule with linear warmup."""
51
+ from torch.optim.lr_scheduler import CosineAnnealingLR, LinearLR, SequentialLR
52
+ warmup = LinearLR(self.optimizer, start_factor=0.1, total_iters=warmup_epochs)
53
+ cosine = CosineAnnealingLR(self.optimizer, T_max=epochs - warmup_epochs, eta_min=1e-6)
54
+ self.scheduler = SequentialLR(self.optimizer, [warmup, cosine], milestones=[warmup_epochs])
55
+
56
+ def train_epoch(self, train_loader: DataLoader) -> float:
57
+ self.model.train()
58
+ total_loss = 0.0
59
+ n_batches = 0
60
+ total_batches = len(train_loader)
61
+
62
+ for batch_idx, (mels, tokens, mel_lengths, token_lengths) in enumerate(train_loader):
63
+ mels = mels.to(self.device)
64
+ tokens = tokens.to(self.device)
65
+ mel_lengths = mel_lengths.to(self.device)
66
+ token_lengths = token_lengths.to(self.device)
67
+
68
+ self.optimizer.zero_grad()
69
+
70
+ log_probs = self.model(mels) # (T', B, vocab)
71
+ output_lengths = self.model.get_output_lengths(mel_lengths)
72
+
73
+ loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
74
+
75
+ if torch.isfinite(loss):
76
+ loss.backward()
77
+ torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=5.0)
78
+ self.optimizer.step()
79
+ total_loss += loss.item()
80
+ n_batches += 1
81
+
82
+ if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == total_batches:
83
+ avg = total_loss / max(n_batches, 1)
84
+ print(f" batch {batch_idx+1}/{total_batches} loss: {avg:.4f}", flush=True)
85
+
86
+ return total_loss / max(n_batches, 1)
87
+
88
+ @torch.no_grad()
89
+ def validate(self, val_loader: DataLoader) -> float:
90
+ self.model.eval()
91
+ total_loss = 0.0
92
+ n_batches = 0
93
+
94
+ for mels, tokens, mel_lengths, token_lengths in val_loader:
95
+ mels = mels.to(self.device)
96
+ tokens = tokens.to(self.device)
97
+ mel_lengths = mel_lengths.to(self.device)
98
+ token_lengths = token_lengths.to(self.device)
99
+
100
+ log_probs = self.model(mels)
101
+ output_lengths = self.model.get_output_lengths(mel_lengths)
102
+
103
+ loss = self.ctc_loss(log_probs, tokens, output_lengths, token_lengths)
104
+ if torch.isfinite(loss):
105
+ total_loss += loss.item()
106
+ n_batches += 1
107
+
108
+ return total_loss / max(n_batches, 1)
109
+
110
+ def train_loop(
111
+ self,
112
+ train_loader: DataLoader,
113
+ val_loader: DataLoader | None,
114
+ epochs: int,
115
+ callback: object | None = None,
116
+ ) -> list[TrainMetrics]:
117
+ for epoch in range(1, epochs + 1):
118
+ train_loss = self.train_epoch(train_loader)
119
+ val_loss = self.validate(val_loader) if val_loader else None
120
+
121
+ metrics = TrainMetrics(epoch=epoch, train_loss=train_loss, val_loss=val_loss)
122
+ self.history.append(metrics)
123
+
124
+ if callback and hasattr(callback, "on_epoch_end"):
125
+ callback.on_epoch_end(metrics)
126
+
127
+ if self.scheduler is not None:
128
+ self.scheduler.step()
129
+
130
+ return self.history
131
+
132
+ @torch.no_grad()
133
+ def greedy_decode(self, mel: torch.Tensor) -> torch.Tensor:
134
+ """Greedy CTC decoding for a single mel spectrogram."""
135
+ self.model.eval()
136
+ if mel.dim() == 2:
137
+ mel = mel.unsqueeze(0) # add batch dim
138
+ mel = mel.to(self.device)
139
+ log_probs = self.model(mel) # (T', 1, vocab)
140
+ pred = log_probs.argmax(dim=-1).squeeze(1) # (T',)
141
+ return pred
src/services/upload_service.py ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ UploadThing upload service — uploads files directly via signed URLs.
3
+
4
+ Generates file keys locally (Sqids + djb2), signs the upload URL with HMAC-SHA256,
5
+ and PUTs directly to the regional ingest endpoint. No server-side API call needed.
6
+
7
+ Requires UPLOADTHING_TOKEN env var (base64-encoded JSON with apiKey, appId, regions).
8
+ """
9
+
10
+ import base64
11
+ import hashlib
12
+ import hmac
13
+ import json
14
+ import math
15
+ import os
16
+ import time
17
+ import urllib.error
18
+ import urllib.request
19
+ import uuid
20
+
21
+ from sqids import Sqids
22
+ from sqids.constants import DEFAULT_ALPHABET
23
+
24
+
25
+ def _djb2(s: str) -> int:
26
+ """DJB2 hash matching UploadThing's JS implementation."""
27
+ h = 5381
28
+ for ch in reversed(s):
29
+ h = (h * 33) ^ ord(ch)
30
+ h &= 0xFFFFFFFF
31
+ h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)
32
+ if h >= 0x80000000:
33
+ h -= 0x100000000
34
+ return h
35
+
36
+
37
+ def _shuffle(input_str: str, seed: str) -> str:
38
+ """Shuffle alphabet deterministically using seed (matches UploadThing JS)."""
39
+ chars = list(input_str)
40
+ seed_num = _djb2(seed)
41
+ for i in range(len(chars)):
42
+ j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars)))
43
+ chars[i], chars[j] = chars[j], chars[i]
44
+ return "".join(chars)
45
+
46
+
47
+ def _generate_file_key(app_id: str) -> str:
48
+ """Generate a unique UploadThing file key using Sqids (matches JS SDK)."""
49
+ alphabet = _shuffle(DEFAULT_ALPHABET, app_id)
50
+ sqids = Sqids(alphabet=alphabet, min_length=12)
51
+ encoded_app_id = sqids.encode([abs(_djb2(app_id))])
52
+ file_seed = base64.urlsafe_b64encode(uuid.uuid4().bytes).decode().rstrip("=")
53
+ return encoded_app_id + file_seed
54
+
55
+
56
+ def _pct_encode(s: str) -> str:
57
+ """Percent-encode like JS encodeURIComponent."""
58
+ return urllib.request.quote(s, safe="")
59
+
60
+
61
+ def _generate_signed_url(
62
+ base_url: str, api_key: str, app_id: str,
63
+ file_name: str, file_size: int,
64
+ file_type: str = "application/octet-stream",
65
+ ) -> str:
66
+ """Generate HMAC-signed upload URL."""
67
+ expires = int(time.time() * 1000) + 3600 * 1000 # 1 hour
68
+
69
+ params = (
70
+ f"?expires={expires}"
71
+ f"&x-ut-identifier={_pct_encode(app_id)}"
72
+ f"&x-ut-file-name={_pct_encode(file_name)}"
73
+ f"&x-ut-file-size={file_size}"
74
+ f"&x-ut-file-type={_pct_encode(file_type)}"
75
+ f"&x-ut-content-disposition=inline"
76
+ f"&x-ut-acl=public-read"
77
+ )
78
+ url = base_url + params
79
+ sig = hmac.new(api_key.encode(), url.encode(), hashlib.sha256).hexdigest()
80
+ return f"{url}&signature=hmac-sha256={sig}"
81
+
82
+
83
+ def _decode_token() -> tuple[str, str, str]:
84
+ """Decode UPLOADTHING_TOKEN to get (api_key, app_id, region)."""
85
+ token = os.environ.get("UPLOADTHING_TOKEN")
86
+ if not token:
87
+ raise RuntimeError("UPLOADTHING_TOKEN env var not set")
88
+ decoded = json.loads(base64.b64decode(token))
89
+ return decoded["apiKey"], decoded["appId"], decoded.get("regions", ["sea1"])[0]
90
+
91
+
92
+ def _build_multipart(file_name: str, file_data: bytes, content_type: str) -> tuple[bytes, str]:
93
+ """Build multipart/form-data body manually (no requests dependency)."""
94
+ boundary = uuid.uuid4().hex
95
+ body = (
96
+ f"--{boundary}\r\n"
97
+ f'Content-Disposition: form-data; name="file"; filename="{file_name}"\r\n'
98
+ f"Content-Type: {content_type}\r\n"
99
+ f"\r\n"
100
+ ).encode() + file_data + f"\r\n--{boundary}--\r\n".encode()
101
+ return body, f"multipart/form-data; boundary={boundary}"
102
+
103
+
104
+ def upload_file(file_path: str, custom_id: str | None = None) -> str | None:
105
+ """
106
+ Upload a single file to UploadThing.
107
+
108
+ Returns the ufsUrl of the uploaded file, or None on failure.
109
+ """
110
+ if not os.path.isfile(file_path):
111
+ print(f" [upload] File not found: {file_path}", flush=True)
112
+ return None
113
+
114
+ api_key, app_id, region = _decode_token()
115
+ file_name = os.path.basename(file_path)
116
+ file_size = os.path.getsize(file_path)
117
+
118
+ # Generate file key and signed URL
119
+ file_key = _generate_file_key(app_id)
120
+ ingest_base = f"https://{region}.ingest.uploadthing.com/{file_key}"
121
+ signed_url = _generate_signed_url(
122
+ ingest_base, api_key, app_id, file_name, file_size,
123
+ )
124
+
125
+ print(f" [upload] Uploading {file_name} ({file_size / 1e6:.1f} MB)...", flush=True)
126
+
127
+ with open(file_path, "rb") as f:
128
+ file_data = f.read()
129
+
130
+ # PUT with multipart form-data (matching the working Python/C# implementations)
131
+ body, content_type = _build_multipart(file_name, file_data, "application/octet-stream")
132
+
133
+ put_req = urllib.request.Request(
134
+ signed_url,
135
+ data=body,
136
+ headers={
137
+ "Content-Type": content_type,
138
+ "Range": "bytes=0-",
139
+ },
140
+ method="PUT",
141
+ )
142
+
143
+ try:
144
+ with urllib.request.urlopen(put_req, timeout=300) as resp:
145
+ resp_body = resp.read().decode()
146
+ resp_json = json.loads(resp_body)
147
+ ufs_url = resp_json.get("ufsUrl") or resp_json.get("url", "")
148
+ print(f" [upload] Success: {ufs_url}", flush=True)
149
+ return ufs_url
150
+ except urllib.error.HTTPError as e:
151
+ body_text = e.read().decode() if e.fp else ""
152
+ print(f" [upload] PUT failed: {e.code} {body_text[:500]}", flush=True)
153
+ return None
154
+ except Exception as e:
155
+ print(f" [upload] PUT error: {e}", flush=True)
156
+ return None
157
+
158
+
159
+ def upload_directory(dir_path: str, prefix: str = "") -> list[str]:
160
+ """Upload all files in a directory. Returns list of uploaded URLs."""
161
+ urls = []
162
+ if not os.path.isdir(dir_path):
163
+ print(f" [upload] Directory not found: {dir_path}", flush=True)
164
+ return urls
165
+
166
+ for fname in sorted(os.listdir(dir_path)):
167
+ fpath = os.path.join(dir_path, fname)
168
+ if os.path.isfile(fpath):
169
+ url = upload_file(fpath, custom_id=f"{prefix}/{fname}" if prefix else fname)
170
+ if url:
171
+ urls.append(url)
172
+
173
+ return urls
src/views/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .console_view import ConsoleView
src/views/console_view.py ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Console view: prints training progress and results."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ if TYPE_CHECKING:
8
+ from src.models.config import GrassmannConfig
9
+ from src.services.training_service import TrainMetrics
10
+
11
+
12
+ class ConsoleView:
13
+ """Simple console-based display for training progress."""
14
+
15
+ def show_model_info(self, config: GrassmannConfig, param_count: int) -> None:
16
+ mode = "Seq2Seq (Encoder-Decoder)" if config.use_seq2seq else "CTC (Encoder-only)"
17
+ print("\n" + "=" * 60)
18
+ print(f" Grassmann Transcription Model — {mode}")
19
+ print("=" * 60)
20
+ print(f" Encoder layers: {config.n_layers}")
21
+ if config.use_seq2seq:
22
+ print(f" Decoder layers: {config.n_decoder_layers}")
23
+ print(f" Cross window: {config.cross_window}")
24
+ print(f" d_model: {config.d_model}")
25
+ print(f" r_reduced: {config.r_reduced}")
26
+ print(f" Plücker dim: {config.plucker_dim}")
27
+ print(f" Window offsets: {config.window_offsets}")
28
+ print(f" Vocab size: {config.vocab_size}")
29
+ print(f" Parameters: {param_count:,}")
30
+ print(f" Attention: NONE (fully Grassmann)")
31
+ print("=" * 60 + "\n")
32
+
33
+ def on_epoch_end(self, metrics: TrainMetrics) -> None:
34
+ val_str = f" val_loss: {metrics.val_loss:.4f}" if metrics.val_loss is not None else ""
35
+ print(f" Epoch {metrics.epoch:3d} | train_loss: {metrics.train_loss:.4f}{val_str}")
36
+
37
+ def show_training_complete(self, history: list[TrainMetrics]) -> None:
38
+ if history:
39
+ best = min(history, key=lambda m: m.val_loss if m.val_loss is not None else m.train_loss)
40
+ print(f"\n Best epoch: {best.epoch} | loss: {best.val_loss or best.train_loss:.4f}")
41
+ print(" Training complete.\n")
42
+
43
+ def show_test_results(self, results: dict) -> None:
44
+ print("\n Test Results:")
45
+ for k, v in results.items():
46
+ print(f" {k}: {v:.4f}")
47
+ print()
48
+
49
+ def show_message(self, msg: str) -> None:
50
+ print(f" {msg}")