Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache β ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download app.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/app.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/app.py
-
curl -L -o app.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/app.py
21.5 kB
| #!/usr/bin/env python3 | |
| """ | |
| Grassmann Transcription β Unified CLI | |
| One entry point for training, caching, evaluation, export, and inference. | |
| Based on "Attention Is Not What You Need" (arXiv:2512.19428). | |
| Usage: | |
| python app.py train --from-cache tisaconundrum/grassmann-mel-cache | |
| python app.py train --recipe 1750h | |
| python app.py cache --recipe 1750h --repo tisaconundrum/grassmann-mel-cache | |
| python app.py export --model checkpoints/bundle --output onnx_export/ | |
| python app.py eval --model eval_model/best_model.pt --config eval_model/config.json | |
| python app.py transcribe --model checkpoints/bundle --audio speech.wav | |
| python app.py demo | |
| """ | |
| import argparse | |
| import os | |
| import sys | |
| import time | |
| import threading | |
| import torch | |
| os.environ["PYTHONUNBUFFERED"] = "1" | |
| sys.stdout.reconfigure(line_buffering=True) | |
| # ββ Shared utilities βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def _start_health_server(port: int = 7860) -> None: | |
| from http.server import HTTPServer, BaseHTTPRequestHandler | |
| class Handler(BaseHTTPRequestHandler): | |
| def do_GET(self): | |
| self.send_response(200) | |
| self.send_header("Content-Type", "text/plain") | |
| self.end_headers() | |
| self.wfile.write(b"Running...") | |
| def log_message(self, format, *args): | |
| pass | |
| server = HTTPServer(("0.0.0.0", port), Handler) | |
| threading.Thread(target=server.serve_forever, daemon=True).start() | |
| print(f" Health server on port {port}", flush=True) | |
| def _log_memory(label: str) -> None: | |
| try: | |
| import resource | |
| rss_kb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss | |
| rss_mb = rss_kb / (1024 * 1024 if sys.platform == "darwin" else 1024) | |
| print(f" [mem] {label}: RSS={rss_mb:.0f}MB", end="", flush=True) | |
| except Exception: | |
| print(f" [mem] {label}: RSS=unknown", end="", flush=True) | |
| if torch.cuda.is_available(): | |
| alloc = torch.cuda.memory_allocated() / 1e6 | |
| reserved = torch.cuda.memory_reserved() / 1e6 | |
| print(f" GPU_alloc={alloc:.0f}MB GPU_reserved={reserved:.0f}MB", flush=True) | |
| else: | |
| print(flush=True) | |
| def _add_model_args(p: argparse.ArgumentParser) -> None: | |
| p.add_argument("--layers", type=int, default=6) | |
| p.add_argument("--d-model", type=int, default=256) | |
| p.add_argument("--r-reduced", type=int, default=32) | |
| p.add_argument("--lr", type=float, default=3e-4) | |
| p.add_argument("--decoder-layers", type=int, default=4) | |
| p.add_argument("--cross-window", type=int, default=8) | |
| p.add_argument("--ctc", action="store_true", help="CTC-only mode") | |
| p.add_argument("--conformer", action="store_true", help="Conformer blocks") | |
| p.add_argument("--two-sided", action="store_true", help="Bidirectional mixing") | |
| p.add_argument("--conv-kernel", type=int, default=31) | |
| p.add_argument("--ablation", action="store_true", help="Symmetric-product ablation (no Grassmann geometry)") | |
| def _auto_pause_space() -> None: | |
| try: | |
| from huggingface_hub import HfApi | |
| space_id = os.environ.get("SPACE_ID") | |
| if space_id: | |
| HfApi().pause_space(space_id) | |
| print(f" Space {space_id} paused.", flush=True) | |
| except Exception as e: | |
| print(f" Note: could not auto-pause: {e}", flush=True) | |
| # ββ train ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_train(args: argparse.Namespace) -> None: | |
| import gc | |
| from src.models.config import GrassmannConfig | |
| from src.controllers.model_controller import ModelController | |
| from src.services.tokenizer_service import CharTokenizer | |
| from src.repositories.dataset_repository import DatasetRepository | |
| from src.repositories.cache_repository import CacheRepository | |
| from src.services.dataset_service import DatasetService | |
| _start_health_server() | |
| config = GrassmannConfig( | |
| epochs=args.epochs, batch_size=args.batch_size, | |
| n_layers=args.layers, n_decoder_layers=args.decoder_layers, | |
| cross_window=args.cross_window, d_model=args.d_model, | |
| r_reduced=args.r_reduced, learning_rate=args.lr, | |
| use_seq2seq=not args.ctc, use_conformer=args.conformer, | |
| two_sided=args.two_sided, conv_kernel_size=args.conv_kernel, | |
| use_ablation=args.ablation, | |
| ) | |
| print("\n" + "=" * 60) | |
| print(" Grassmann Transcription β Training") | |
| print("=" * 60) | |
| if torch.cuda.is_available(): | |
| gpu = torch.cuda.get_device_name(0) | |
| mem = torch.cuda.get_device_properties(0).total_memory / 1e9 | |
| print(f" GPU: {gpu} ({mem:.1f} GB)") | |
| else: | |
| print(" GPU: None (CPU)") | |
| mode = "CTC" if args.ctc else "Seq2Seq" | |
| block = "Conformer" if args.conformer else "Plain" | |
| mix = "two-sided" if args.two_sided else "causal" | |
| mixer = "Ablation (symmetric)" if args.ablation else "Grassmann (PlΓΌcker)" | |
| print(f" Mode: {mode} | Blocks: {block} ({mix})") | |
| print(f" Mixer: {mixer}") | |
| print(f" Encoder: {args.layers}L / d={args.d_model} / r={args.r_reduced}") | |
| print("=" * 60 + "\n") | |
| tokenizer = CharTokenizer() | |
| cache_dir = os.path.join(args.save_dir, "mel_cache") | |
| repo = DatasetRepository(config, tokenizer) | |
| cache = CacheRepository(cache_dir) | |
| dataset = DatasetService(repo, cache) | |
| if args.from_cache: | |
| dataset.load_from_hub_cache(args.from_cache).clean().preprocess().split( | |
| max_samples=args.max_samples) | |
| elif args.recipe: | |
| dataset.load_recipe(args.recipe, | |
| max_samples_per_source=args.max_per_source).clean().preprocess().split() | |
| else: | |
| dataset.load_dataset(args.dataset, split=args.split, | |
| max_samples=args.max_samples).clean().preprocess().split() | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| _log_memory("After dataset load") | |
| model_ctrl = ModelController(config, tokenizer) | |
| model_ctrl.build() | |
| assert model_ctrl.trainer is not None | |
| model_ctrl.trainer.set_cosine_schedule(args.epochs, warmup_epochs=min(3, args.epochs // 5)) | |
| train_loader = dataset.get_train_loader(config.batch_size) | |
| val_loader = dataset.get_val_loader(config.batch_size) | |
| os.makedirs(args.save_dir, exist_ok=True) | |
| best_val_loss = float("inf") | |
| start_time = time.time() | |
| _log_memory("Before training") | |
| for epoch in range(1, args.epochs + 1): | |
| epoch_start = time.time() | |
| train_loss = model_ctrl.trainer.train_epoch(train_loader) | |
| val_loss = model_ctrl.trainer.validate(val_loader) | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| torch.cuda.empty_cache() | |
| if model_ctrl.trainer.scheduler is not None: | |
| model_ctrl.trainer.scheduler.step() | |
| lr = model_ctrl.trainer.optimizer.param_groups[0]["lr"] | |
| elapsed = time.time() - epoch_start | |
| total_elapsed = time.time() - start_time | |
| mem_str = "" | |
| if torch.cuda.is_available(): | |
| alloc = torch.cuda.memory_allocated() / 1e9 | |
| reserved = torch.cuda.memory_reserved() / 1e9 | |
| mem_str = f" GPU: {alloc:.1f}/{reserved:.1f}GB" | |
| print(f" Epoch {epoch:3d}/{args.epochs} | " | |
| f"train: {train_loss:.4f} val: {val_loss:.4f} " | |
| f"lr: {lr:.2e} ({elapsed:.1f}s, total: {total_elapsed/60:.1f}min)" | |
| f"{mem_str}", flush=True) | |
| if val_loss < best_val_loss: | |
| best_val_loss = val_loss | |
| assert model_ctrl.nn_model is not None | |
| torch.save(model_ctrl.nn_model.state_dict(), | |
| os.path.join(args.save_dir, "best_model.pt")) | |
| print(f" -> Saved best model (val_loss={val_loss:.4f})", flush=True) | |
| if epoch % 10 == 0: | |
| assert model_ctrl.nn_model is not None | |
| torch.save(model_ctrl.nn_model.state_dict(), | |
| os.path.join(args.save_dir, f"epoch_{epoch}.pt")) | |
| if epoch % args.decode_every == 0 or epoch == 1: | |
| _decode_samples(model_ctrl, dataset) | |
| assert model_ctrl.nn_model is not None | |
| torch.save(model_ctrl.nn_model.state_dict(), os.path.join(args.save_dir, "final_model.pt")) | |
| test_loader = dataset.get_test_loader(config.batch_size) | |
| test_loss = model_ctrl.trainer.validate(test_loader) | |
| total_time = time.time() - start_time | |
| print(f"\n{'=' * 60}") | |
| print(f" Training Complete!") | |
| print(f" Best val loss: {best_val_loss:.4f}") | |
| print(f" Test loss: {test_loss:.4f}") | |
| print(f" Total time: {total_time/60:.1f} minutes") | |
| print(f"{'=' * 60}") | |
| _decode_samples(model_ctrl, dataset, n=5) | |
| _post_training(model_ctrl, config, args.save_dir, run_name=args.run_name) | |
| def _decode_samples(model_ctrl, dataset, n: int = 3) -> None: | |
| assert model_ctrl.trainer is not None and dataset._val_ds is not None | |
| print("\n Sample transcriptions:", flush=True) | |
| for i in range(min(n, len(dataset._val_ds))): | |
| mel, tokens = dataset._val_ds[i] | |
| pred_text = model_ctrl.predict(mel) | |
| gt_text = model_ctrl.tokenizer.decode(tokens) | |
| print(f" [{i}] GT: '{gt_text}'", flush=True) | |
| print(f" [{i}] PRED: '{pred_text}'") | |
| print() | |
| def _post_training(model_ctrl, config, save_dir: str, run_name: str | None = None) -> None: | |
| from src.services.export_service import export_seq2seq, export_ctc | |
| def _prefixed(name: str) -> str: | |
| """Prefix a filename with run_name so uploads are distinguishable.""" | |
| if not run_name: | |
| return name | |
| base, ext = os.path.splitext(name) | |
| return f"{run_name}-{base}{ext}" | |
| try: | |
| from src.services.model_hub_service import save_model | |
| assert model_ctrl.nn_model is not None | |
| save_model(model_ctrl.nn_model, config, os.path.join(save_dir, "bundle")) | |
| print(" Model bundle saved.", flush=True) | |
| except Exception as e: | |
| print(f" Warning: bundle save failed: {e}", flush=True) | |
| try: | |
| onnx_dir = os.path.join(save_dir, "onnx") | |
| if config.use_seq2seq: | |
| export_seq2seq(model_ctrl.nn_model, config, onnx_dir) | |
| else: | |
| export_ctc(model_ctrl.nn_model, config, onnx_dir) | |
| print(f" ONNX exported to {onnx_dir}/", flush=True) | |
| except Exception as e: | |
| print(f" Warning: ONNX export failed: {e}", flush=True) | |
| # Rename artifacts with run_name prefix before upload | |
| if run_name: | |
| import shutil | |
| # Rename best_model.pt β ablation-smol-best_model.pt etc. | |
| for name in ["best_model.pt", "final_model.pt"]: | |
| src = os.path.join(save_dir, name) | |
| dst = os.path.join(save_dir, _prefixed(name)) | |
| if os.path.isfile(src): | |
| shutil.move(src, dst) | |
| # Rename ONNX files (model.onnx β ablation-smol-model.onnx etc.) | |
| onnx_dir = os.path.join(save_dir, "onnx") | |
| if os.path.isdir(onnx_dir): | |
| for fname in list(os.listdir(onnx_dir)): | |
| src = os.path.join(onnx_dir, fname) | |
| dst = os.path.join(onnx_dir, _prefixed(fname)) | |
| if os.path.isfile(src): | |
| shutil.move(src, dst) | |
| # Rename bundle config files | |
| bundle_dir = os.path.join(save_dir, "bundle") | |
| if os.path.isdir(bundle_dir): | |
| for fname in list(os.listdir(bundle_dir)): | |
| src = os.path.join(bundle_dir, fname) | |
| dst = os.path.join(bundle_dir, _prefixed(fname)) | |
| if os.path.isfile(src): | |
| shutil.move(src, dst) | |
| try: | |
| from src.services.upload_service import upload_file, upload_directory | |
| bundle_dir = os.path.join(save_dir, "bundle") | |
| onnx_dir = os.path.join(save_dir, "onnx") | |
| if os.path.isdir(bundle_dir): | |
| for fname in sorted(os.listdir(bundle_dir)): | |
| fpath = os.path.join(bundle_dir, fname) | |
| if os.path.isfile(fpath): | |
| upload_file(fpath) | |
| if os.path.isdir(onnx_dir): | |
| urls = upload_directory(onnx_dir) | |
| print(f" Uploaded {len(urls)} ONNX files", flush=True) | |
| best_name = _prefixed("best_model.pt") | |
| best_path = os.path.join(save_dir, best_name) | |
| if os.path.isfile(best_path): | |
| upload_file(best_path) | |
| except Exception as e: | |
| print(f" Warning: upload failed: {e}", flush=True) | |
| _auto_pause_space() | |
| # ββ cache ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_cache(args: argparse.Namespace) -> None: | |
| from src.models.config import GrassmannConfig | |
| from src.services.tokenizer_service import CharTokenizer | |
| from src.repositories.dataset_repository import DatasetRepository | |
| from src.repositories.cache_repository import CacheRepository | |
| from src.services.dataset_service import DatasetService | |
| _start_health_server() | |
| print("\n" + "=" * 60) | |
| print(" Grassmann β Dataset Caching (CPU)") | |
| print("=" * 60) | |
| print(f" Recipe: {args.recipe}") | |
| print(f" Upload to: {args.repo}") | |
| print("=" * 60 + "\n") | |
| config = GrassmannConfig() | |
| tokenizer = CharTokenizer() | |
| repo = DatasetRepository(config, tokenizer) | |
| cache = CacheRepository(args.cache_dir) | |
| dataset = DatasetService(repo, cache) | |
| start = time.time() | |
| dataset.load_recipe(args.recipe, | |
| max_samples_per_source=args.max_per_source, | |
| upload_repo=args.repo, | |
| workers=args.workers) | |
| print(f"\n Caching complete in {(time.time() - start)/60:.1f} minutes") | |
| _auto_pause_space() | |
| print(f"\n Done! Now train with: python app.py train --from-cache {args.repo}") | |
| # ββ export βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_export(args: argparse.Namespace) -> None: | |
| import json | |
| from dataclasses import asdict | |
| from pathlib import Path | |
| from src.services.model_hub_service import load_model | |
| from src.services.export_service import export_seq2seq, export_ctc, verify_onnx | |
| print(f"\n Loading model from {args.model}...") | |
| model, config, tokenizer = load_model(args.model, device="cpu") | |
| print(f" {sum(p.numel() for p in model.parameters()):,} parameters") | |
| print(f" Mode: {'seq2seq' if config.use_seq2seq else 'CTC'}") | |
| if config.use_seq2seq: | |
| export_seq2seq(model, config, args.output, args.opset) | |
| else: | |
| export_ctc(model, config, args.output, args.opset) | |
| out = Path(args.output) | |
| with open(out / "config.json", "w") as f: | |
| json.dump(asdict(config), f, indent=2) | |
| with open(out / "tokenizer_info.json", "w") as f: | |
| json.dump({"type": "char", "vocab_size": tokenizer.vocab_size}, f, indent=2) | |
| print(f"\n Export complete!") | |
| if args.verify: | |
| verify_onnx(args.output, config, config.use_seq2seq) | |
| # ββ eval βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_eval(args: argparse.Namespace) -> None: | |
| import json | |
| from src.models.config import GrassmannConfig | |
| from src.models.grassmann_model import GrassmannTranscriptionModel | |
| from src.services.tokenizer_service import CharTokenizer | |
| from src.services.eval_service import evaluate_wer | |
| print(" Loading model...", flush=True) | |
| with open(args.config) as f: | |
| config_dict = json.load(f) | |
| config = GrassmannConfig(**{ | |
| k: v for k, v in config_dict.items() | |
| if k in GrassmannConfig.__dataclass_fields__ | |
| }) | |
| tokenizer = CharTokenizer() | |
| config.vocab_size = tokenizer.vocab_size | |
| model = GrassmannTranscriptionModel(config) | |
| model.load_state_dict(torch.load(args.model, map_location="cpu", weights_only=True)) | |
| model.eval() | |
| print(f" {sum(p.numel() for p in model.parameters()):,} parameters") | |
| evaluate_wer(model, config, tokenizer, split=args.split, | |
| max_samples=args.max_samples, show_errors=args.show_errors) | |
| # ββ transcribe βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_transcribe(args: argparse.Namespace) -> None: | |
| from src.models.config import GrassmannConfig | |
| from src.controllers.pipeline_controller import PipelineController | |
| config = GrassmannConfig() | |
| pipeline = PipelineController(config) | |
| pipeline.load_model(args.model) | |
| text = pipeline.transcribe(args.audio) | |
| print(f"\n {text}\n") | |
| # ββ demo βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def cmd_demo() -> None: | |
| from src.models.config import GrassmannConfig | |
| from src.controllers.model_controller import ModelController | |
| from src.services.tokenizer_service import CharTokenizer | |
| print("\n Running demo with synthetic data...\n") | |
| config = GrassmannConfig(n_layers=2, d_model=128, r_reduced=16, d_ff=512, | |
| n_decoder_layers=2, epochs=3, batch_size=4) | |
| tokenizer = CharTokenizer() | |
| config.vocab_size = tokenizer.vocab_size | |
| model = ModelController(config, tokenizer) | |
| model.build() | |
| assert model.trainer is not None | |
| device = model.trainer.device | |
| mels = torch.randn(4, config.n_mels, 200, device=device) | |
| assert model.nn_model is not None | |
| if config.use_seq2seq: | |
| tokens = tokenizer.encode("hello world").unsqueeze(0).expand(4, -1) | |
| logits = model.nn_model(mels, tokens) | |
| print(f" Forward pass OK β output: {logits.shape}") | |
| else: | |
| log_probs = model.nn_model(mels) | |
| print(f" Forward pass OK β output: {log_probs.shape}") | |
| pred_tokens = model.trainer.greedy_decode(mels[0:1]) | |
| decoded = tokenizer.decode(pred_tokens) | |
| print(f" Greedy decode (untrained): '{decoded}'") | |
| print("\n Demo complete!\n") | |
| # ββ CLI ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="Grassmann Manifold Audio Transcription") | |
| sub = parser.add_subparsers(dest="command") | |
| # train | |
| p = sub.add_parser("train", help="Train model (GPU)") | |
| p.add_argument("--from-cache", default=None, help="HF Dataset repo with pre-cached mels") | |
| p.add_argument("--recipe", default=None, help="Multi-dataset recipe (e.g. '1750h')") | |
| p.add_argument("--dataset", default="openslr/librispeech_asr") | |
| p.add_argument("--split", default="train.clean.100") | |
| p.add_argument("--max-samples", type=int, default=None) | |
| p.add_argument("--max-per-source", type=int, default=None) | |
| p.add_argument("--epochs", type=int, default=50) | |
| p.add_argument("--batch-size", type=int, default=32) | |
| p.add_argument("--save-dir", default="checkpoints") | |
| p.add_argument("--run-name", default=None, help="Prefix for uploaded artifacts (e.g. 'ablation-smol')") | |
| p.add_argument("--decode-every", type=int, default=5) | |
| _add_model_args(p) | |
| # cache | |
| p = sub.add_parser("cache", help="Cache dataset mels to HF Hub (CPU)") | |
| p.add_argument("--recipe", required=True) | |
| p.add_argument("--repo", required=True, help="HF Dataset repo") | |
| p.add_argument("--cache-dir", default="mel_cache") | |
| p.add_argument("--max-per-source", type=int, default=None) | |
| p.add_argument("--workers", type=int, default=0, help="Parallel mel workers (0=sequential, default)") | |
| # export | |
| p = sub.add_parser("export", help="Export model to ONNX") | |
| p.add_argument("--model", required=True) | |
| p.add_argument("--output", default="onnx_export") | |
| p.add_argument("--opset", type=int, default=18) | |
| p.add_argument("--verify", action="store_true") | |
| # eval | |
| p = sub.add_parser("eval", help="Evaluate WER/CER on LibriSpeech") | |
| p.add_argument("--model", default="eval_model/best_model.pt") | |
| p.add_argument("--config", default="eval_model/config.json") | |
| p.add_argument("--split", default="test.clean") | |
| p.add_argument("--max-samples", type=int, default=None) | |
| p.add_argument("--show-errors", type=int, default=10) | |
| # transcribe | |
| p = sub.add_parser("transcribe", help="Transcribe audio file") | |
| p.add_argument("--model", required=True) | |
| p.add_argument("--audio", required=True) | |
| # demo | |
| sub.add_parser("demo", help="Sanity check with synthetic data") | |
| args = parser.parse_args() | |
| commands = { | |
| "train": cmd_train, "cache": cmd_cache, "export": cmd_export, | |
| "eval": cmd_eval, "transcribe": cmd_transcribe, "demo": lambda a: cmd_demo(), | |
| } | |
| if args.command in commands: | |
| commands[args.command](args) | |
| else: | |
| parser.print_help() | |
| sys.exit(1) | |
| if __name__ == "__main__": | |
| main() | |