grassmann-cache / app.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache β€” ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
21.5 kB
#!/usr/bin/env python3
"""
Grassmann Transcription β€” Unified CLI
One entry point for training, caching, evaluation, export, and inference.
Based on "Attention Is Not What You Need" (arXiv:2512.19428).
Usage:
python app.py train --from-cache tisaconundrum/grassmann-mel-cache
python app.py train --recipe 1750h
python app.py cache --recipe 1750h --repo tisaconundrum/grassmann-mel-cache
python app.py export --model checkpoints/bundle --output onnx_export/
python app.py eval --model eval_model/best_model.pt --config eval_model/config.json
python app.py transcribe --model checkpoints/bundle --audio speech.wav
python app.py demo
"""
import argparse
import os
import sys
import time
import threading
import torch
os.environ["PYTHONUNBUFFERED"] = "1"
sys.stdout.reconfigure(line_buffering=True)
# ── Shared utilities ─────────────────────────────────────────────────────
def _start_health_server(port: int = 7860) -> None:
from http.server import HTTPServer, BaseHTTPRequestHandler
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.end_headers()
self.wfile.write(b"Running...")
def log_message(self, format, *args):
pass
server = HTTPServer(("0.0.0.0", port), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
print(f" Health server on port {port}", flush=True)
def _log_memory(label: str) -> None:
try:
import resource
rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
rss_mb = rss_kb / (1024 * 1024 if sys.platform == "darwin" else 1024)
print(f" [mem] {label}: RSS={rss_mb:.0f}MB", end="", flush=True)
except Exception:
print(f" [mem] {label}: RSS=unknown", end="", flush=True)
if torch.cuda.is_available():
alloc = torch.cuda.memory_allocated() / 1e6
reserved = torch.cuda.memory_reserved() / 1e6
print(f" GPU_alloc={alloc:.0f}MB GPU_reserved={reserved:.0f}MB", flush=True)
else:
print(flush=True)
def _add_model_args(p: argparse.ArgumentParser) -> None:
p.add_argument("--layers", type=int, default=6)
p.add_argument("--d-model", type=int, default=256)
p.add_argument("--r-reduced", type=int, default=32)
p.add_argument("--lr", type=float, default=3e-4)
p.add_argument("--decoder-layers", type=int, default=4)
p.add_argument("--cross-window", type=int, default=8)
p.add_argument("--ctc", action="store_true", help="CTC-only mode")
p.add_argument("--conformer", action="store_true", help="Conformer blocks")
p.add_argument("--two-sided", action="store_true", help="Bidirectional mixing")
p.add_argument("--conv-kernel", type=int, default=31)
p.add_argument("--ablation", action="store_true", help="Symmetric-product ablation (no Grassmann geometry)")
def _auto_pause_space() -> None:
try:
from huggingface_hub import HfApi
space_id = os.environ.get("SPACE_ID")
if space_id:
HfApi().pause_space(space_id)
print(f" Space {space_id} paused.", flush=True)
except Exception as e:
print(f" Note: could not auto-pause: {e}", flush=True)
# ── train ────────────────────────────────────────────────────────────────
def cmd_train(args: argparse.Namespace) -> None:
import gc
from src.models.config import GrassmannConfig
from src.controllers.model_controller import ModelController
from src.services.tokenizer_service import CharTokenizer
from src.repositories.dataset_repository import DatasetRepository
from src.repositories.cache_repository import CacheRepository
from src.services.dataset_service import DatasetService
_start_health_server()
config = GrassmannConfig(
epochs=args.epochs, batch_size=args.batch_size,
n_layers=args.layers, n_decoder_layers=args.decoder_layers,
cross_window=args.cross_window, d_model=args.d_model,
r_reduced=args.r_reduced, learning_rate=args.lr,
use_seq2seq=not args.ctc, use_conformer=args.conformer,
two_sided=args.two_sided, conv_kernel_size=args.conv_kernel,
use_ablation=args.ablation,
)
print("\n" + "=" * 60)
print(" Grassmann Transcription β€” Training")
print("=" * 60)
if torch.cuda.is_available():
gpu = torch.cuda.get_device_name(0)
mem = torch.cuda.get_device_properties(0).total_memory / 1e9
print(f" GPU: {gpu} ({mem:.1f} GB)")
else:
print(" GPU: None (CPU)")
mode = "CTC" if args.ctc else "Seq2Seq"
block = "Conformer" if args.conformer else "Plain"
mix = "two-sided" if args.two_sided else "causal"
mixer = "Ablation (symmetric)" if args.ablation else "Grassmann (PlΓΌcker)"
print(f" Mode: {mode} | Blocks: {block} ({mix})")
print(f" Mixer: {mixer}")
print(f" Encoder: {args.layers}L / d={args.d_model} / r={args.r_reduced}")
print("=" * 60 + "\n")
tokenizer = CharTokenizer()
cache_dir = os.path.join(args.save_dir, "mel_cache")
repo = DatasetRepository(config, tokenizer)
cache = CacheRepository(cache_dir)
dataset = DatasetService(repo, cache)
if args.from_cache:
dataset.load_from_hub_cache(args.from_cache).clean().preprocess().split(
max_samples=args.max_samples)
elif args.recipe:
dataset.load_recipe(args.recipe,
max_samples_per_source=args.max_per_source).clean().preprocess().split()
else:
dataset.load_dataset(args.dataset, split=args.split,
max_samples=args.max_samples).clean().preprocess().split()
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
_log_memory("After dataset load")
model_ctrl = ModelController(config, tokenizer)
model_ctrl.build()
assert model_ctrl.trainer is not None
model_ctrl.trainer.set_cosine_schedule(args.epochs, warmup_epochs=min(3, args.epochs // 5))
train_loader = dataset.get_train_loader(config.batch_size)
val_loader = dataset.get_val_loader(config.batch_size)
os.makedirs(args.save_dir, exist_ok=True)
best_val_loss = float("inf")
start_time = time.time()
_log_memory("Before training")
for epoch in range(1, args.epochs + 1):
epoch_start = time.time()
train_loss = model_ctrl.trainer.train_epoch(train_loader)
val_loss = model_ctrl.trainer.validate(val_loader)
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
if model_ctrl.trainer.scheduler is not None:
model_ctrl.trainer.scheduler.step()
lr = model_ctrl.trainer.optimizer.param_groups[0]["lr"]
elapsed = time.time() - epoch_start
total_elapsed = time.time() - start_time
mem_str = ""
if torch.cuda.is_available():
alloc = torch.cuda.memory_allocated() / 1e9
reserved = torch.cuda.memory_reserved() / 1e9
mem_str = f" GPU: {alloc:.1f}/{reserved:.1f}GB"
print(f" Epoch {epoch:3d}/{args.epochs} | "
f"train: {train_loss:.4f} val: {val_loss:.4f} "
f"lr: {lr:.2e} ({elapsed:.1f}s, total: {total_elapsed/60:.1f}min)"
f"{mem_str}", flush=True)
if val_loss < best_val_loss:
best_val_loss = val_loss
assert model_ctrl.nn_model is not None
torch.save(model_ctrl.nn_model.state_dict(),
os.path.join(args.save_dir, "best_model.pt"))
print(f" -> Saved best model (val_loss={val_loss:.4f})", flush=True)
if epoch % 10 == 0:
assert model_ctrl.nn_model is not None
torch.save(model_ctrl.nn_model.state_dict(),
os.path.join(args.save_dir, f"epoch_{epoch}.pt"))
if epoch % args.decode_every == 0 or epoch == 1:
_decode_samples(model_ctrl, dataset)
assert model_ctrl.nn_model is not None
torch.save(model_ctrl.nn_model.state_dict(), os.path.join(args.save_dir, "final_model.pt"))
test_loader = dataset.get_test_loader(config.batch_size)
test_loss = model_ctrl.trainer.validate(test_loader)
total_time = time.time() - start_time
print(f"\n{'=' * 60}")
print(f" Training Complete!")
print(f" Best val loss: {best_val_loss:.4f}")
print(f" Test loss: {test_loss:.4f}")
print(f" Total time: {total_time/60:.1f} minutes")
print(f"{'=' * 60}")
_decode_samples(model_ctrl, dataset, n=5)
_post_training(model_ctrl, config, args.save_dir, run_name=args.run_name)
def _decode_samples(model_ctrl, dataset, n: int = 3) -> None:
assert model_ctrl.trainer is not None and dataset._val_ds is not None
print("\n Sample transcriptions:", flush=True)
for i in range(min(n, len(dataset._val_ds))):
mel, tokens = dataset._val_ds[i]
pred_text = model_ctrl.predict(mel)
gt_text = model_ctrl.tokenizer.decode(tokens)
print(f" [{i}] GT: '{gt_text}'", flush=True)
print(f" [{i}] PRED: '{pred_text}'")
print()
def _post_training(model_ctrl, config, save_dir: str, run_name: str | None = None) -> None:
from src.services.export_service import export_seq2seq, export_ctc
def _prefixed(name: str) -> str:
"""Prefix a filename with run_name so uploads are distinguishable."""
if not run_name:
return name
base, ext = os.path.splitext(name)
return f"{run_name}-{base}{ext}"
try:
from src.services.model_hub_service import save_model
assert model_ctrl.nn_model is not None
save_model(model_ctrl.nn_model, config, os.path.join(save_dir, "bundle"))
print(" Model bundle saved.", flush=True)
except Exception as e:
print(f" Warning: bundle save failed: {e}", flush=True)
try:
onnx_dir = os.path.join(save_dir, "onnx")
if config.use_seq2seq:
export_seq2seq(model_ctrl.nn_model, config, onnx_dir)
else:
export_ctc(model_ctrl.nn_model, config, onnx_dir)
print(f" ONNX exported to {onnx_dir}/", flush=True)
except Exception as e:
print(f" Warning: ONNX export failed: {e}", flush=True)
# Rename artifacts with run_name prefix before upload
if run_name:
import shutil
# Rename best_model.pt β†’ ablation-smol-best_model.pt etc.
for name in ["best_model.pt", "final_model.pt"]:
src = os.path.join(save_dir, name)
dst = os.path.join(save_dir, _prefixed(name))
if os.path.isfile(src):
shutil.move(src, dst)
# Rename ONNX files (model.onnx β†’ ablation-smol-model.onnx etc.)
onnx_dir = os.path.join(save_dir, "onnx")
if os.path.isdir(onnx_dir):
for fname in list(os.listdir(onnx_dir)):
src = os.path.join(onnx_dir, fname)
dst = os.path.join(onnx_dir, _prefixed(fname))
if os.path.isfile(src):
shutil.move(src, dst)
# Rename bundle config files
bundle_dir = os.path.join(save_dir, "bundle")
if os.path.isdir(bundle_dir):
for fname in list(os.listdir(bundle_dir)):
src = os.path.join(bundle_dir, fname)
dst = os.path.join(bundle_dir, _prefixed(fname))
if os.path.isfile(src):
shutil.move(src, dst)
try:
from src.services.upload_service import upload_file, upload_directory
bundle_dir = os.path.join(save_dir, "bundle")
onnx_dir = os.path.join(save_dir, "onnx")
if os.path.isdir(bundle_dir):
for fname in sorted(os.listdir(bundle_dir)):
fpath = os.path.join(bundle_dir, fname)
if os.path.isfile(fpath):
upload_file(fpath)
if os.path.isdir(onnx_dir):
urls = upload_directory(onnx_dir)
print(f" Uploaded {len(urls)} ONNX files", flush=True)
best_name = _prefixed("best_model.pt")
best_path = os.path.join(save_dir, best_name)
if os.path.isfile(best_path):
upload_file(best_path)
except Exception as e:
print(f" Warning: upload failed: {e}", flush=True)
_auto_pause_space()
# ── cache ────────────────────────────────────────────────────────────────
def cmd_cache(args: argparse.Namespace) -> None:
from src.models.config import GrassmannConfig
from src.services.tokenizer_service import CharTokenizer
from src.repositories.dataset_repository import DatasetRepository
from src.repositories.cache_repository import CacheRepository
from src.services.dataset_service import DatasetService
_start_health_server()
print("\n" + "=" * 60)
print(" Grassmann β€” Dataset Caching (CPU)")
print("=" * 60)
print(f" Recipe: {args.recipe}")
print(f" Upload to: {args.repo}")
print("=" * 60 + "\n")
config = GrassmannConfig()
tokenizer = CharTokenizer()
repo = DatasetRepository(config, tokenizer)
cache = CacheRepository(args.cache_dir)
dataset = DatasetService(repo, cache)
start = time.time()
dataset.load_recipe(args.recipe,
max_samples_per_source=args.max_per_source,
upload_repo=args.repo,
workers=args.workers)
print(f"\n Caching complete in {(time.time() - start)/60:.1f} minutes")
_auto_pause_space()
print(f"\n Done! Now train with: python app.py train --from-cache {args.repo}")
# ── export ───────────────────────────────────────────────────────────────
def cmd_export(args: argparse.Namespace) -> None:
import json
from dataclasses import asdict
from pathlib import Path
from src.services.model_hub_service import load_model
from src.services.export_service import export_seq2seq, export_ctc, verify_onnx
print(f"\n Loading model from {args.model}...")
model, config, tokenizer = load_model(args.model, device="cpu")
print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
print(f" Mode: {'seq2seq' if config.use_seq2seq else 'CTC'}")
if config.use_seq2seq:
export_seq2seq(model, config, args.output, args.opset)
else:
export_ctc(model, config, args.output, args.opset)
out = Path(args.output)
with open(out / "config.json", "w") as f:
json.dump(asdict(config), f, indent=2)
with open(out / "tokenizer_info.json", "w") as f:
json.dump({"type": "char", "vocab_size": tokenizer.vocab_size}, f, indent=2)
print(f"\n Export complete!")
if args.verify:
verify_onnx(args.output, config, config.use_seq2seq)
# ── eval ─────────────────────────────────────────────────────────────────
def cmd_eval(args: argparse.Namespace) -> None:
import json
from src.models.config import GrassmannConfig
from src.models.grassmann_model import GrassmannTranscriptionModel
from src.services.tokenizer_service import CharTokenizer
from src.services.eval_service import evaluate_wer
print(" Loading model...", flush=True)
with open(args.config) as f:
config_dict = json.load(f)
config = GrassmannConfig(**{
k: v for k, v in config_dict.items()
if k in GrassmannConfig.__dataclass_fields__
})
tokenizer = CharTokenizer()
config.vocab_size = tokenizer.vocab_size
model = GrassmannTranscriptionModel(config)
model.load_state_dict(torch.load(args.model, map_location="cpu", weights_only=True))
model.eval()
print(f" {sum(p.numel() for p in model.parameters()):,} parameters")
evaluate_wer(model, config, tokenizer, split=args.split,
max_samples=args.max_samples, show_errors=args.show_errors)
# ── transcribe ───────────────────────────────────────────────────────────
def cmd_transcribe(args: argparse.Namespace) -> None:
from src.models.config import GrassmannConfig
from src.controllers.pipeline_controller import PipelineController
config = GrassmannConfig()
pipeline = PipelineController(config)
pipeline.load_model(args.model)
text = pipeline.transcribe(args.audio)
print(f"\n {text}\n")
# ── demo ─────────────────────────────────────────────────────────────────
def cmd_demo() -> None:
from src.models.config import GrassmannConfig
from src.controllers.model_controller import ModelController
from src.services.tokenizer_service import CharTokenizer
print("\n Running demo with synthetic data...\n")
config = GrassmannConfig(n_layers=2, d_model=128, r_reduced=16, d_ff=512,
n_decoder_layers=2, epochs=3, batch_size=4)
tokenizer = CharTokenizer()
config.vocab_size = tokenizer.vocab_size
model = ModelController(config, tokenizer)
model.build()
assert model.trainer is not None
device = model.trainer.device
mels = torch.randn(4, config.n_mels, 200, device=device)
assert model.nn_model is not None
if config.use_seq2seq:
tokens = tokenizer.encode("hello world").unsqueeze(0).expand(4, -1)
logits = model.nn_model(mels, tokens)
print(f" Forward pass OK β€” output: {logits.shape}")
else:
log_probs = model.nn_model(mels)
print(f" Forward pass OK β€” output: {log_probs.shape}")
pred_tokens = model.trainer.greedy_decode(mels[0:1])
decoded = tokenizer.decode(pred_tokens)
print(f" Greedy decode (untrained): '{decoded}'")
print("\n Demo complete!\n")
# ── CLI ──────────────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(description="Grassmann Manifold Audio Transcription")
sub = parser.add_subparsers(dest="command")
# train
p = sub.add_parser("train", help="Train model (GPU)")
p.add_argument("--from-cache", default=None, help="HF Dataset repo with pre-cached mels")
p.add_argument("--recipe", default=None, help="Multi-dataset recipe (e.g. '1750h')")
p.add_argument("--dataset", default="openslr/librispeech_asr")
p.add_argument("--split", default="train.clean.100")
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--max-per-source", type=int, default=None)
p.add_argument("--epochs", type=int, default=50)
p.add_argument("--batch-size", type=int, default=32)
p.add_argument("--save-dir", default="checkpoints")
p.add_argument("--run-name", default=None, help="Prefix for uploaded artifacts (e.g. 'ablation-smol')")
p.add_argument("--decode-every", type=int, default=5)
_add_model_args(p)
# cache
p = sub.add_parser("cache", help="Cache dataset mels to HF Hub (CPU)")
p.add_argument("--recipe", required=True)
p.add_argument("--repo", required=True, help="HF Dataset repo")
p.add_argument("--cache-dir", default="mel_cache")
p.add_argument("--max-per-source", type=int, default=None)
p.add_argument("--workers", type=int, default=0, help="Parallel mel workers (0=sequential, default)")
# export
p = sub.add_parser("export", help="Export model to ONNX")
p.add_argument("--model", required=True)
p.add_argument("--output", default="onnx_export")
p.add_argument("--opset", type=int, default=18)
p.add_argument("--verify", action="store_true")
# eval
p = sub.add_parser("eval", help="Evaluate WER/CER on LibriSpeech")
p.add_argument("--model", default="eval_model/best_model.pt")
p.add_argument("--config", default="eval_model/config.json")
p.add_argument("--split", default="test.clean")
p.add_argument("--max-samples", type=int, default=None)
p.add_argument("--show-errors", type=int, default=10)
# transcribe
p = sub.add_parser("transcribe", help="Transcribe audio file")
p.add_argument("--model", required=True)
p.add_argument("--audio", required=True)
# demo
sub.add_parser("demo", help="Sanity check with synthetic data")
args = parser.parse_args()
commands = {
"train": cmd_train, "cache": cmd_cache, "export": cmd_export,
"eval": cmd_eval, "transcribe": cmd_transcribe, "demo": lambda a: cmd_demo(),
}
if args.command in commands:
commands[args.command](args)
else:
parser.print_help()
sys.exit(1)
if __name__ == "__main__":
main()