Instructions to use cowWhySo/AuroraASR-1M-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use cowWhySo/AuroraASR-1M-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download cowWhySo/AuroraASR-1M-MLX --local-dir AuroraASR-1M-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download scripts/bench_native.py from cowWhySo/AuroraASR-1M-MLX: direct link, hf CLI and curl.
- Browser
- Download file 12.6 kB
-
https://huggingface.co/cowWhySo/AuroraASR-1M-MLX/resolve/main/scripts/bench_native.py
- Command line
-
hf download hf://cowWhySo/AuroraASR-1M-MLX/scripts/bench_native.py
-
curl -L -o bench_native.py https://huggingface.co/cowWhySo/AuroraASR-1M-MLX/resolve/main/scripts/bench_native.py
12.6 kB
| #!/usr/bin/env python3 | |
| """Native MLX benchmark: parity gate, latency variants, length scaling, memory. No ONNX Runtime. | |
| Every latency variant runs in its own fresh process (so peak memory and RSS belong to that | |
| variant alone) and the variants are interleaved over several rounds in shuffled order, which | |
| spreads thermal drift and background load across them instead of favouring whichever ran first. | |
| python scripts/bench_native.py --model-dir . --rounds 3 --runs 60 --out benchmarks/native_mlx_report.json | |
| Apple Silicon macOS only. Cross-backend (ONNX / Core ML) numbers come from the sibling repos. | |
| """ | |
| import argparse | |
| import json | |
| import platform | |
| import random | |
| import resource | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| import numpy as np | |
| ROOT = Path(__file__).resolve().parent.parent # repo root; this file lives in scripts/ | |
| # name -> (compile, output path, uses cached device input?) | |
| VARIANTS = { | |
| 'eager_logits_host': dict(compile=False, out='logits', cached=False), # first-release path | |
| 'compiled_logits_host': dict(compile=True, out='logits', cached=False), | |
| 'eager_ids_host': dict(compile=False, out='ids', cached=False), | |
| 'compiled_ids_host': dict(compile=True, out='ids', cached=False), # recommended | |
| 'compiled_device_only': dict(compile=True, out='device', cached=True), # diagnostic: no host traffic | |
| } | |
| PIPELINES = {'pipeline_numpy_frontend': 'numpy', 'pipeline_mlx_frontend': 'mlx'} | |
| def pctl(samples): | |
| a = np.asarray(samples) | |
| return {'p50_ms': float(np.median(a)), 'p95_ms': float(np.percentile(a, 95)), | |
| 'mean_ms': float(a.mean()), 'min_ms': float(a.min()), 'runs': int(a.size)} | |
| def mib(n): | |
| return n / 2**20 | |
| # ----------------------------------------------------------------------------- worker (one process) | |
| def worker(args): | |
| rss_import = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss | |
| import mlx.core as mx | |
| sys.path.insert(0, str(ROOT)) | |
| from aurora_pipeline import Transcriber, read_audio | |
| name, runs, warm = args.worker, args.runs, args.warmup | |
| model_dir = Path(args.model_dir) | |
| fix = np.load(model_dir / 'reference_fixture.npz') | |
| x = np.ascontiguousarray(fix['acoustic_input'], dtype=np.float32) | |
| expected = fix['reference_logits'] | |
| mx.reset_peak_memory() | |
| if name in VARIANTS: | |
| v = VARIANTS[name] | |
| asr = Transcriber(model_dir, compile=v['compile']) | |
| x_dev = mx.array(x) | |
| mx.eval(x_dev) | |
| if v['out'] == 'logits': | |
| def once(inp=None): | |
| return np.asarray(asr.logits(x if inp is None else inp)) | |
| elif v['out'] == 'ids': | |
| def once(inp=None): | |
| return asr.ids(x if inp is None else inp) | |
| else: | |
| def once(inp=None): | |
| mx.eval(asr._logits(x_dev)) | |
| parity = None | |
| if v['out'] != 'device': | |
| y = np.asarray(asr.logits(x)) | |
| parity = {'max_abs': float(np.abs(y - expected).max()), | |
| 'argmax_agreement': float((y.argmax(-1) == expected.argmax(-1)).mean())} | |
| for _ in range(warm): | |
| once() | |
| else: | |
| asr = Transcriber(model_dir, compile=True, frontend=PIPELINES[name]) | |
| wave = read_audio(model_dir / 'reference_jfk.flac') | |
| parity = {'transcript': asr.transcribe(wave)} | |
| def once(inp=None): | |
| return asr.transcribe(wave) | |
| for _ in range(warm): | |
| once() | |
| times = [] | |
| for _ in range(runs): | |
| t = time.perf_counter_ns() | |
| once() | |
| times.append((time.perf_counter_ns() - t) / 1e6) | |
| out = {'name': name, 'latency': pctl(times), 'parity': parity, | |
| 'mlx_peak_memory_mib': mib(mx.get_peak_memory()), | |
| 'mlx_active_memory_mib': mib(mx.get_active_memory()), | |
| 'mlx_cache_memory_mib': mib(mx.get_cache_memory()), | |
| 'process_peak_rss_mib': mib(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss), | |
| 'rss_after_imports_only_mib': mib(rss_import)} | |
| print(json.dumps(out)) | |
| def scaling_worker(args): | |
| """Latency vs clip length and batch size, compiled id path, JFK tiled to the target length.""" | |
| sys.path.insert(0, str(ROOT)) | |
| from aurora_pipeline import SAMPLE_RATE, Transcriber, read_audio | |
| model_dir = Path(args.model_dir) | |
| asr = Transcriber(model_dir, compile=True) | |
| base = read_audio(model_dir / 'reference_jfk.flac') | |
| rows = [] | |
| for seconds in (2, 5.5, 11, 22, 44, 88): | |
| wave = np.tile(base, int(np.ceil(seconds * SAMPLE_RATE / base.size)))[:int(seconds * SAMPLE_RATE)] | |
| feats = asr.features(wave) | |
| for _ in range(8): | |
| asr.ids(feats) | |
| ts = [] | |
| for _ in range(max(10, args.runs // 3)): | |
| t = time.perf_counter_ns() | |
| asr.ids(feats) | |
| ts.append((time.perf_counter_ns() - t) / 1e6) | |
| p50 = float(np.median(ts)) | |
| rows.append({'audio_seconds': seconds, 'mel_frames': int(feats.shape[1]), 'acoustic_p50_ms': p50, | |
| 'rtf': p50 / (seconds * 1000), 'x_realtime': seconds * 1000 / p50}) | |
| batch_rows = [] | |
| f11 = np.asarray(asr.features(base)[0]) | |
| for b in (1, 4, 8, 16): | |
| batch = np.repeat(f11[None], b, axis=0) | |
| for _ in range(8): | |
| asr.ids(batch) | |
| ts = [] | |
| for _ in range(max(10, args.runs // 3)): | |
| t = time.perf_counter_ns() | |
| asr.ids(batch) | |
| ts.append((time.perf_counter_ns() - t) / 1e6) | |
| p50 = float(np.median(ts)) | |
| batch_rows.append({'batch': b, 'acoustic_p50_ms': p50, 'ms_per_utterance': p50 / b, | |
| 'utterances_per_s': b * 1000 / p50}) | |
| print(json.dumps({'length_scaling': rows, 'batch_scaling_11s': batch_rows})) | |
| def parity_worker(args): | |
| """Reference-logit parity per length (torch inputs), then waveform -> logits for each frontend.""" | |
| sys.path.insert(0, str(ROOT)) | |
| from aurora_pipeline import Transcriber | |
| model_dir = Path(args.model_dir) | |
| fix = np.load(model_dir / 'parity_fixtures.npz') | |
| meta = json.loads(bytes(fix['meta_json']).decode()) | |
| asr = {fe: Transcriber(model_dir, compile=True, frontend=fe) for fe in ('numpy', 'mlx')} | |
| cases = [k for k in meta['cases'] if k != 'batch'] | |
| rows = [] | |
| for name in cases: | |
| ref = fix[f'{name}__logits'] | |
| y = np.asarray(asr['numpy'].logits(fix[f'{name}__input'])) | |
| d = np.abs(y - ref) | |
| row = {'case': name, 'mel_frames': meta['cases'][name]['mel_frames'], 'logit_frames': int(y.shape[1]), | |
| 'torch_features': {'max_abs': float(d.max()), 'mean_abs': float(d.mean()), | |
| 'p99_abs': float(np.percentile(d, 99)), | |
| 'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()), | |
| 'text_match': asr['numpy'].decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}} | |
| for fe, a in asr.items(): # full path from the waveform, using this repo's own frontend | |
| y = np.asarray(a.logits(a.features(fix[f'{name}__waveform']))) | |
| d = np.abs(y - ref) | |
| row[f'waveform_{fe}_frontend'] = { | |
| 'max_abs': float(d.max()), 'mean_abs': float(d.mean()), | |
| 'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()), | |
| 'text_match': a.decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']} | |
| rows.append(row) | |
| x, n = fix['batch__input'], fix['batch__lengths'] | |
| ref, out_len = fix['batch__logits'], fix['batch__output_lengths'] | |
| y = np.asarray(asr['numpy'].logits(x, n)) | |
| batch = {'items': [], 'input_shape': list(x.shape)} | |
| for i, L in enumerate(out_len): | |
| d = np.abs(y[i, :L] - ref[i, :L]) | |
| batch['items'].append({'valid_logit_frames': int(L), 'max_abs': float(d.max()), | |
| 'argmax_agreement': float((y[i, :L].argmax(-1) == ref[i, :L].argmax(-1)).mean())}) | |
| print(json.dumps({'torch': meta['torch'], 'per_length': rows, 'padded_batch': batch})) | |
| # ----------------------------------------------------------------------------- driver | |
| def spawn(args, name, extra=()): | |
| cmd = [sys.executable, '-I', str(Path(__file__).resolve()), '--model-dir', str(args.model_dir), | |
| '--runs', str(args.runs), '--warmup', str(args.warmup), *extra, '--worker', name] | |
| res = subprocess.run(cmd, capture_output=True, text=True) | |
| if res.returncode: | |
| raise RuntimeError(f'{name} failed:\n{res.stderr}') | |
| return json.loads(res.stdout.strip().splitlines()[-1]) | |
| def driver(args): | |
| if platform.system() != 'Darwin' or platform.machine() != 'arm64': | |
| raise SystemExit('Run on Apple Silicon macOS.') | |
| names = list(VARIANTS) + list(PIPELINES) | |
| rng = random.Random(1337) | |
| results = {n: [] for n in names} | |
| for r in range(args.rounds): | |
| order = names[:] | |
| rng.shuffle(order) | |
| for n in order: | |
| results[n].append(spawn(args, n)) | |
| print(f'round {r + 1}/{args.rounds} done', file=sys.stderr) | |
| summary = {} | |
| for n, rs in results.items(): | |
| # Pool per-round medians; a round's own p50/p95 are kept so run-to-run spread is visible. | |
| p50s = [r['latency']['p50_ms'] for r in rs] | |
| summary[n] = { | |
| 'p50_ms_median_of_rounds': float(np.median(p50s)), | |
| 'p50_ms_per_round': p50s, | |
| 'p95_ms_median_of_rounds': float(np.median([r['latency']['p95_ms'] for r in rs])), | |
| 'runs_per_round': rs[0]['latency']['runs'], 'rounds': len(rs), | |
| 'parity': rs[0]['parity'], | |
| 'mlx_peak_memory_mib': max(r['mlx_peak_memory_mib'] for r in rs), | |
| 'mlx_active_memory_mib': max(r['mlx_active_memory_mib'] for r in rs), | |
| 'mlx_cache_memory_mib': max(r['mlx_cache_memory_mib'] for r in rs), | |
| 'process_peak_rss_mib': max(r['process_peak_rss_mib'] for r in rs), | |
| 'rss_after_imports_only_mib': min(r['rss_after_imports_only_mib'] for r in rs), | |
| } | |
| base = summary['eager_logits_host']['p50_ms_median_of_rounds'] | |
| for n, s in summary.items(): | |
| s['speedup_vs_eager_logits_host'] = base / s['p50_ms_median_of_rounds'] if n in VARIANTS else None | |
| report = { | |
| 'platform': platform.platform(), 'machine': platform.machine(), | |
| 'cpu': subprocess.run(['sysctl', '-n', 'machdep.cpu.brand_string'], capture_output=True, text=True).stdout.strip(), | |
| 'python': platform.python_version(), | |
| 'input': {'mel_frames': 1101, 'audio_seconds': 11.0, 'fixture': 'reference_fixture.npz'}, | |
| 'protocol': f'{args.rounds} rounds x {args.runs} timed runs after {args.warmup} warmup, each variant in a ' | |
| 'fresh process, order shuffled per round', | |
| 'variants': summary, | |
| 'parity_multi_length': spawn(args, 'parity'), | |
| 'scaling': spawn(args, 'scaling'), | |
| 'notes': ['host-output variants include host-to-device upload of the 1101x80 features and device-to-host copy of the result', | |
| 'process_peak_rss_mib includes the Python/MLX runtime; rss_after_imports_only_mib is the same process before the model loads', | |
| 'mlx_* memory is the MLX allocator; mlx_cache_memory_mib is reusable freed buffers, not live tensors', | |
| 'single machine, single clip; no WER evaluation']} | |
| import mlx.core as mx | |
| report['mlx_version'] = getattr(mx, '__version__', None) or subprocess.run( | |
| [sys.executable, '-c', 'import importlib.metadata as m;print(m.version("mlx"))'], | |
| capture_output=True, text=True).stdout.strip() | |
| out = Path(args.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(report, indent=2) + '\n') | |
| print(json.dumps({n: (round(s['p50_ms_median_of_rounds'], 3), round(s['p95_ms_median_of_rounds'], 3), | |
| round(s['process_peak_rss_mib'], 1)) for n, s in summary.items()}, indent=2)) | |
| print(f'saved {out}') | |
| if __name__ == '__main__': | |
| p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| p.add_argument('--model-dir', default=str(ROOT)) | |
| p.add_argument('--runs', type=int, default=60) | |
| p.add_argument('--warmup', type=int, default=20) | |
| p.add_argument('--rounds', type=int, default=3) | |
| p.add_argument('--out', default=str(ROOT / 'benchmarks' / 'native_mlx_report.json')) | |
| p.add_argument('--worker', help=argparse.SUPPRESS) | |
| a = p.parse_args() | |
| if a.worker == 'scaling': | |
| scaling_worker(a) | |
| elif a.worker == 'parity': | |
| parity_worker(a) | |
| elif a.worker: | |
| worker(a) | |
| else: | |
| driver(a) | |