Instructions to use cowWhySo/AuroraASR-1M-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use cowWhySo/AuroraASR-1M-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download cowWhySo/AuroraASR-1M-MLX --local-dir AuroraASR-1M-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 12,629 Bytes
a78c443 2d44270 a78c443 2d44270 a78c443 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 | #!/usr/bin/env python3
"""Native MLX benchmark: parity gate, latency variants, length scaling, memory. No ONNX Runtime.
Every latency variant runs in its own fresh process (so peak memory and RSS belong to that
variant alone) and the variants are interleaved over several rounds in shuffled order, which
spreads thermal drift and background load across them instead of favouring whichever ran first.
python scripts/bench_native.py --model-dir . --rounds 3 --runs 60 --out benchmarks/native_mlx_report.json
Apple Silicon macOS only. Cross-backend (ONNX / Core ML) numbers come from the sibling repos.
"""
import argparse
import json
import platform
import random
import resource
import subprocess
import sys
import time
from pathlib import Path
import numpy as np
ROOT = Path(__file__).resolve().parent.parent # repo root; this file lives in scripts/
# name -> (compile, output path, uses cached device input?)
VARIANTS = {
'eager_logits_host': dict(compile=False, out='logits', cached=False), # first-release path
'compiled_logits_host': dict(compile=True, out='logits', cached=False),
'eager_ids_host': dict(compile=False, out='ids', cached=False),
'compiled_ids_host': dict(compile=True, out='ids', cached=False), # recommended
'compiled_device_only': dict(compile=True, out='device', cached=True), # diagnostic: no host traffic
}
PIPELINES = {'pipeline_numpy_frontend': 'numpy', 'pipeline_mlx_frontend': 'mlx'}
def pctl(samples):
a = np.asarray(samples)
return {'p50_ms': float(np.median(a)), 'p95_ms': float(np.percentile(a, 95)),
'mean_ms': float(a.mean()), 'min_ms': float(a.min()), 'runs': int(a.size)}
def mib(n):
return n / 2**20
# ----------------------------------------------------------------------------- worker (one process)
def worker(args):
rss_import = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
import mlx.core as mx
sys.path.insert(0, str(ROOT))
from aurora_pipeline import Transcriber, read_audio
name, runs, warm = args.worker, args.runs, args.warmup
model_dir = Path(args.model_dir)
fix = np.load(model_dir / 'reference_fixture.npz')
x = np.ascontiguousarray(fix['acoustic_input'], dtype=np.float32)
expected = fix['reference_logits']
mx.reset_peak_memory()
if name in VARIANTS:
v = VARIANTS[name]
asr = Transcriber(model_dir, compile=v['compile'])
x_dev = mx.array(x)
mx.eval(x_dev)
if v['out'] == 'logits':
def once(inp=None):
return np.asarray(asr.logits(x if inp is None else inp))
elif v['out'] == 'ids':
def once(inp=None):
return asr.ids(x if inp is None else inp)
else:
def once(inp=None):
mx.eval(asr._logits(x_dev))
parity = None
if v['out'] != 'device':
y = np.asarray(asr.logits(x))
parity = {'max_abs': float(np.abs(y - expected).max()),
'argmax_agreement': float((y.argmax(-1) == expected.argmax(-1)).mean())}
for _ in range(warm):
once()
else:
asr = Transcriber(model_dir, compile=True, frontend=PIPELINES[name])
wave = read_audio(model_dir / 'reference_jfk.flac')
parity = {'transcript': asr.transcribe(wave)}
def once(inp=None):
return asr.transcribe(wave)
for _ in range(warm):
once()
times = []
for _ in range(runs):
t = time.perf_counter_ns()
once()
times.append((time.perf_counter_ns() - t) / 1e6)
out = {'name': name, 'latency': pctl(times), 'parity': parity,
'mlx_peak_memory_mib': mib(mx.get_peak_memory()),
'mlx_active_memory_mib': mib(mx.get_active_memory()),
'mlx_cache_memory_mib': mib(mx.get_cache_memory()),
'process_peak_rss_mib': mib(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
'rss_after_imports_only_mib': mib(rss_import)}
print(json.dumps(out))
def scaling_worker(args):
"""Latency vs clip length and batch size, compiled id path, JFK tiled to the target length."""
sys.path.insert(0, str(ROOT))
from aurora_pipeline import SAMPLE_RATE, Transcriber, read_audio
model_dir = Path(args.model_dir)
asr = Transcriber(model_dir, compile=True)
base = read_audio(model_dir / 'reference_jfk.flac')
rows = []
for seconds in (2, 5.5, 11, 22, 44, 88):
wave = np.tile(base, int(np.ceil(seconds * SAMPLE_RATE / base.size)))[:int(seconds * SAMPLE_RATE)]
feats = asr.features(wave)
for _ in range(8):
asr.ids(feats)
ts = []
for _ in range(max(10, args.runs // 3)):
t = time.perf_counter_ns()
asr.ids(feats)
ts.append((time.perf_counter_ns() - t) / 1e6)
p50 = float(np.median(ts))
rows.append({'audio_seconds': seconds, 'mel_frames': int(feats.shape[1]), 'acoustic_p50_ms': p50,
'rtf': p50 / (seconds * 1000), 'x_realtime': seconds * 1000 / p50})
batch_rows = []
f11 = np.asarray(asr.features(base)[0])
for b in (1, 4, 8, 16):
batch = np.repeat(f11[None], b, axis=0)
for _ in range(8):
asr.ids(batch)
ts = []
for _ in range(max(10, args.runs // 3)):
t = time.perf_counter_ns()
asr.ids(batch)
ts.append((time.perf_counter_ns() - t) / 1e6)
p50 = float(np.median(ts))
batch_rows.append({'batch': b, 'acoustic_p50_ms': p50, 'ms_per_utterance': p50 / b,
'utterances_per_s': b * 1000 / p50})
print(json.dumps({'length_scaling': rows, 'batch_scaling_11s': batch_rows}))
def parity_worker(args):
"""Reference-logit parity per length (torch inputs), then waveform -> logits for each frontend."""
sys.path.insert(0, str(ROOT))
from aurora_pipeline import Transcriber
model_dir = Path(args.model_dir)
fix = np.load(model_dir / 'parity_fixtures.npz')
meta = json.loads(bytes(fix['meta_json']).decode())
asr = {fe: Transcriber(model_dir, compile=True, frontend=fe) for fe in ('numpy', 'mlx')}
cases = [k for k in meta['cases'] if k != 'batch']
rows = []
for name in cases:
ref = fix[f'{name}__logits']
y = np.asarray(asr['numpy'].logits(fix[f'{name}__input']))
d = np.abs(y - ref)
row = {'case': name, 'mel_frames': meta['cases'][name]['mel_frames'], 'logit_frames': int(y.shape[1]),
'torch_features': {'max_abs': float(d.max()), 'mean_abs': float(d.mean()),
'p99_abs': float(np.percentile(d, 99)),
'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()),
'text_match': asr['numpy'].decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}}
for fe, a in asr.items(): # full path from the waveform, using this repo's own frontend
y = np.asarray(a.logits(a.features(fix[f'{name}__waveform'])))
d = np.abs(y - ref)
row[f'waveform_{fe}_frontend'] = {
'max_abs': float(d.max()), 'mean_abs': float(d.mean()),
'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()),
'text_match': a.decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}
rows.append(row)
x, n = fix['batch__input'], fix['batch__lengths']
ref, out_len = fix['batch__logits'], fix['batch__output_lengths']
y = np.asarray(asr['numpy'].logits(x, n))
batch = {'items': [], 'input_shape': list(x.shape)}
for i, L in enumerate(out_len):
d = np.abs(y[i, :L] - ref[i, :L])
batch['items'].append({'valid_logit_frames': int(L), 'max_abs': float(d.max()),
'argmax_agreement': float((y[i, :L].argmax(-1) == ref[i, :L].argmax(-1)).mean())})
print(json.dumps({'torch': meta['torch'], 'per_length': rows, 'padded_batch': batch}))
# ----------------------------------------------------------------------------- driver
def spawn(args, name, extra=()):
cmd = [sys.executable, '-I', str(Path(__file__).resolve()), '--model-dir', str(args.model_dir),
'--runs', str(args.runs), '--warmup', str(args.warmup), *extra, '--worker', name]
res = subprocess.run(cmd, capture_output=True, text=True)
if res.returncode:
raise RuntimeError(f'{name} failed:\n{res.stderr}')
return json.loads(res.stdout.strip().splitlines()[-1])
def driver(args):
if platform.system() != 'Darwin' or platform.machine() != 'arm64':
raise SystemExit('Run on Apple Silicon macOS.')
names = list(VARIANTS) + list(PIPELINES)
rng = random.Random(1337)
results = {n: [] for n in names}
for r in range(args.rounds):
order = names[:]
rng.shuffle(order)
for n in order:
results[n].append(spawn(args, n))
print(f'round {r + 1}/{args.rounds} done', file=sys.stderr)
summary = {}
for n, rs in results.items():
# Pool per-round medians; a round's own p50/p95 are kept so run-to-run spread is visible.
p50s = [r['latency']['p50_ms'] for r in rs]
summary[n] = {
'p50_ms_median_of_rounds': float(np.median(p50s)),
'p50_ms_per_round': p50s,
'p95_ms_median_of_rounds': float(np.median([r['latency']['p95_ms'] for r in rs])),
'runs_per_round': rs[0]['latency']['runs'], 'rounds': len(rs),
'parity': rs[0]['parity'],
'mlx_peak_memory_mib': max(r['mlx_peak_memory_mib'] for r in rs),
'mlx_active_memory_mib': max(r['mlx_active_memory_mib'] for r in rs),
'mlx_cache_memory_mib': max(r['mlx_cache_memory_mib'] for r in rs),
'process_peak_rss_mib': max(r['process_peak_rss_mib'] for r in rs),
'rss_after_imports_only_mib': min(r['rss_after_imports_only_mib'] for r in rs),
}
base = summary['eager_logits_host']['p50_ms_median_of_rounds']
for n, s in summary.items():
s['speedup_vs_eager_logits_host'] = base / s['p50_ms_median_of_rounds'] if n in VARIANTS else None
report = {
'platform': platform.platform(), 'machine': platform.machine(),
'cpu': subprocess.run(['sysctl', '-n', 'machdep.cpu.brand_string'], capture_output=True, text=True).stdout.strip(),
'python': platform.python_version(),
'input': {'mel_frames': 1101, 'audio_seconds': 11.0, 'fixture': 'reference_fixture.npz'},
'protocol': f'{args.rounds} rounds x {args.runs} timed runs after {args.warmup} warmup, each variant in a '
'fresh process, order shuffled per round',
'variants': summary,
'parity_multi_length': spawn(args, 'parity'),
'scaling': spawn(args, 'scaling'),
'notes': ['host-output variants include host-to-device upload of the 1101x80 features and device-to-host copy of the result',
'process_peak_rss_mib includes the Python/MLX runtime; rss_after_imports_only_mib is the same process before the model loads',
'mlx_* memory is the MLX allocator; mlx_cache_memory_mib is reusable freed buffers, not live tensors',
'single machine, single clip; no WER evaluation']}
import mlx.core as mx
report['mlx_version'] = getattr(mx, '__version__', None) or subprocess.run(
[sys.executable, '-c', 'import importlib.metadata as m;print(m.version("mlx"))'],
capture_output=True, text=True).stdout.strip()
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({n: (round(s['p50_ms_median_of_rounds'], 3), round(s['p95_ms_median_of_rounds'], 3),
round(s['process_peak_rss_mib'], 1)) for n, s in summary.items()}, indent=2))
print(f'saved {out}')
if __name__ == '__main__':
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('--model-dir', default=str(ROOT))
p.add_argument('--runs', type=int, default=60)
p.add_argument('--warmup', type=int, default=20)
p.add_argument('--rounds', type=int, default=3)
p.add_argument('--out', default=str(ROOT / 'benchmarks' / 'native_mlx_report.json'))
p.add_argument('--worker', help=argparse.SUPPRESS)
a = p.parse_args()
if a.worker == 'scaling':
scaling_worker(a)
elif a.worker == 'parity':
parity_worker(a)
elif a.worker:
worker(a)
else:
driver(a)
|