#!/usr/bin/env python3 """Native MLX benchmark: parity gate, latency variants, length scaling, memory. No ONNX Runtime. Every latency variant runs in its own fresh process (so peak memory and RSS belong to that variant alone) and the variants are interleaved over several rounds in shuffled order, which spreads thermal drift and background load across them instead of favouring whichever ran first. python scripts/bench_native.py --model-dir . --rounds 3 --runs 60 --out benchmarks/native_mlx_report.json Apple Silicon macOS only. Cross-backend (ONNX / Core ML) numbers come from the sibling repos. """ import argparse import json import platform import random import resource import subprocess import sys import time from pathlib import Path import numpy as np ROOT = Path(__file__).resolve().parent.parent # repo root; this file lives in scripts/ # name -> (compile, output path, uses cached device input?) VARIANTS = { 'eager_logits_host': dict(compile=False, out='logits', cached=False), # first-release path 'compiled_logits_host': dict(compile=True, out='logits', cached=False), 'eager_ids_host': dict(compile=False, out='ids', cached=False), 'compiled_ids_host': dict(compile=True, out='ids', cached=False), # recommended 'compiled_device_only': dict(compile=True, out='device', cached=True), # diagnostic: no host traffic } PIPELINES = {'pipeline_numpy_frontend': 'numpy', 'pipeline_mlx_frontend': 'mlx'} def pctl(samples): a = np.asarray(samples) return {'p50_ms': float(np.median(a)), 'p95_ms': float(np.percentile(a, 95)), 'mean_ms': float(a.mean()), 'min_ms': float(a.min()), 'runs': int(a.size)} def mib(n): return n / 2**20 # ----------------------------------------------------------------------------- worker (one process) def worker(args): rss_import = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss import mlx.core as mx sys.path.insert(0, str(ROOT)) from aurora_pipeline import Transcriber, read_audio name, runs, warm = args.worker, args.runs, args.warmup model_dir = Path(args.model_dir) fix = np.load(model_dir / 'reference_fixture.npz') x = np.ascontiguousarray(fix['acoustic_input'], dtype=np.float32) expected = fix['reference_logits'] mx.reset_peak_memory() if name in VARIANTS: v = VARIANTS[name] asr = Transcriber(model_dir, compile=v['compile']) x_dev = mx.array(x) mx.eval(x_dev) if v['out'] == 'logits': def once(inp=None): return np.asarray(asr.logits(x if inp is None else inp)) elif v['out'] == 'ids': def once(inp=None): return asr.ids(x if inp is None else inp) else: def once(inp=None): mx.eval(asr._logits(x_dev)) parity = None if v['out'] != 'device': y = np.asarray(asr.logits(x)) parity = {'max_abs': float(np.abs(y - expected).max()), 'argmax_agreement': float((y.argmax(-1) == expected.argmax(-1)).mean())} for _ in range(warm): once() else: asr = Transcriber(model_dir, compile=True, frontend=PIPELINES[name]) wave = read_audio(model_dir / 'reference_jfk.flac') parity = {'transcript': asr.transcribe(wave)} def once(inp=None): return asr.transcribe(wave) for _ in range(warm): once() times = [] for _ in range(runs): t = time.perf_counter_ns() once() times.append((time.perf_counter_ns() - t) / 1e6) out = {'name': name, 'latency': pctl(times), 'parity': parity, 'mlx_peak_memory_mib': mib(mx.get_peak_memory()), 'mlx_active_memory_mib': mib(mx.get_active_memory()), 'mlx_cache_memory_mib': mib(mx.get_cache_memory()), 'process_peak_rss_mib': mib(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss), 'rss_after_imports_only_mib': mib(rss_import)} print(json.dumps(out)) def scaling_worker(args): """Latency vs clip length and batch size, compiled id path, JFK tiled to the target length.""" sys.path.insert(0, str(ROOT)) from aurora_pipeline import SAMPLE_RATE, Transcriber, read_audio model_dir = Path(args.model_dir) asr = Transcriber(model_dir, compile=True) base = read_audio(model_dir / 'reference_jfk.flac') rows = [] for seconds in (2, 5.5, 11, 22, 44, 88): wave = np.tile(base, int(np.ceil(seconds * SAMPLE_RATE / base.size)))[:int(seconds * SAMPLE_RATE)] feats = asr.features(wave) for _ in range(8): asr.ids(feats) ts = [] for _ in range(max(10, args.runs // 3)): t = time.perf_counter_ns() asr.ids(feats) ts.append((time.perf_counter_ns() - t) / 1e6) p50 = float(np.median(ts)) rows.append({'audio_seconds': seconds, 'mel_frames': int(feats.shape[1]), 'acoustic_p50_ms': p50, 'rtf': p50 / (seconds * 1000), 'x_realtime': seconds * 1000 / p50}) batch_rows = [] f11 = np.asarray(asr.features(base)[0]) for b in (1, 4, 8, 16): batch = np.repeat(f11[None], b, axis=0) for _ in range(8): asr.ids(batch) ts = [] for _ in range(max(10, args.runs // 3)): t = time.perf_counter_ns() asr.ids(batch) ts.append((time.perf_counter_ns() - t) / 1e6) p50 = float(np.median(ts)) batch_rows.append({'batch': b, 'acoustic_p50_ms': p50, 'ms_per_utterance': p50 / b, 'utterances_per_s': b * 1000 / p50}) print(json.dumps({'length_scaling': rows, 'batch_scaling_11s': batch_rows})) def parity_worker(args): """Reference-logit parity per length (torch inputs), then waveform -> logits for each frontend.""" sys.path.insert(0, str(ROOT)) from aurora_pipeline import Transcriber model_dir = Path(args.model_dir) fix = np.load(model_dir / 'parity_fixtures.npz') meta = json.loads(bytes(fix['meta_json']).decode()) asr = {fe: Transcriber(model_dir, compile=True, frontend=fe) for fe in ('numpy', 'mlx')} cases = [k for k in meta['cases'] if k != 'batch'] rows = [] for name in cases: ref = fix[f'{name}__logits'] y = np.asarray(asr['numpy'].logits(fix[f'{name}__input'])) d = np.abs(y - ref) row = {'case': name, 'mel_frames': meta['cases'][name]['mel_frames'], 'logit_frames': int(y.shape[1]), 'torch_features': {'max_abs': float(d.max()), 'mean_abs': float(d.mean()), 'p99_abs': float(np.percentile(d, 99)), 'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()), 'text_match': asr['numpy'].decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}} for fe, a in asr.items(): # full path from the waveform, using this repo's own frontend y = np.asarray(a.logits(a.features(fix[f'{name}__waveform']))) d = np.abs(y - ref) row[f'waveform_{fe}_frontend'] = { 'max_abs': float(d.max()), 'mean_abs': float(d.mean()), 'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()), 'text_match': a.decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']} rows.append(row) x, n = fix['batch__input'], fix['batch__lengths'] ref, out_len = fix['batch__logits'], fix['batch__output_lengths'] y = np.asarray(asr['numpy'].logits(x, n)) batch = {'items': [], 'input_shape': list(x.shape)} for i, L in enumerate(out_len): d = np.abs(y[i, :L] - ref[i, :L]) batch['items'].append({'valid_logit_frames': int(L), 'max_abs': float(d.max()), 'argmax_agreement': float((y[i, :L].argmax(-1) == ref[i, :L].argmax(-1)).mean())}) print(json.dumps({'torch': meta['torch'], 'per_length': rows, 'padded_batch': batch})) # ----------------------------------------------------------------------------- driver def spawn(args, name, extra=()): cmd = [sys.executable, '-I', str(Path(__file__).resolve()), '--model-dir', str(args.model_dir), '--runs', str(args.runs), '--warmup', str(args.warmup), *extra, '--worker', name] res = subprocess.run(cmd, capture_output=True, text=True) if res.returncode: raise RuntimeError(f'{name} failed:\n{res.stderr}') return json.loads(res.stdout.strip().splitlines()[-1]) def driver(args): if platform.system() != 'Darwin' or platform.machine() != 'arm64': raise SystemExit('Run on Apple Silicon macOS.') names = list(VARIANTS) + list(PIPELINES) rng = random.Random(1337) results = {n: [] for n in names} for r in range(args.rounds): order = names[:] rng.shuffle(order) for n in order: results[n].append(spawn(args, n)) print(f'round {r + 1}/{args.rounds} done', file=sys.stderr) summary = {} for n, rs in results.items(): # Pool per-round medians; a round's own p50/p95 are kept so run-to-run spread is visible. p50s = [r['latency']['p50_ms'] for r in rs] summary[n] = { 'p50_ms_median_of_rounds': float(np.median(p50s)), 'p50_ms_per_round': p50s, 'p95_ms_median_of_rounds': float(np.median([r['latency']['p95_ms'] for r in rs])), 'runs_per_round': rs[0]['latency']['runs'], 'rounds': len(rs), 'parity': rs[0]['parity'], 'mlx_peak_memory_mib': max(r['mlx_peak_memory_mib'] for r in rs), 'mlx_active_memory_mib': max(r['mlx_active_memory_mib'] for r in rs), 'mlx_cache_memory_mib': max(r['mlx_cache_memory_mib'] for r in rs), 'process_peak_rss_mib': max(r['process_peak_rss_mib'] for r in rs), 'rss_after_imports_only_mib': min(r['rss_after_imports_only_mib'] for r in rs), } base = summary['eager_logits_host']['p50_ms_median_of_rounds'] for n, s in summary.items(): s['speedup_vs_eager_logits_host'] = base / s['p50_ms_median_of_rounds'] if n in VARIANTS else None report = { 'platform': platform.platform(), 'machine': platform.machine(), 'cpu': subprocess.run(['sysctl', '-n', 'machdep.cpu.brand_string'], capture_output=True, text=True).stdout.strip(), 'python': platform.python_version(), 'input': {'mel_frames': 1101, 'audio_seconds': 11.0, 'fixture': 'reference_fixture.npz'}, 'protocol': f'{args.rounds} rounds x {args.runs} timed runs after {args.warmup} warmup, each variant in a ' 'fresh process, order shuffled per round', 'variants': summary, 'parity_multi_length': spawn(args, 'parity'), 'scaling': spawn(args, 'scaling'), 'notes': ['host-output variants include host-to-device upload of the 1101x80 features and device-to-host copy of the result', 'process_peak_rss_mib includes the Python/MLX runtime; rss_after_imports_only_mib is the same process before the model loads', 'mlx_* memory is the MLX allocator; mlx_cache_memory_mib is reusable freed buffers, not live tensors', 'single machine, single clip; no WER evaluation']} import mlx.core as mx report['mlx_version'] = getattr(mx, '__version__', None) or subprocess.run( [sys.executable, '-c', 'import importlib.metadata as m;print(m.version("mlx"))'], capture_output=True, text=True).stdout.strip() out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(report, indent=2) + '\n') print(json.dumps({n: (round(s['p50_ms_median_of_rounds'], 3), round(s['p95_ms_median_of_rounds'], 3), round(s['process_peak_rss_mib'], 1)) for n, s in summary.items()}, indent=2)) print(f'saved {out}') if __name__ == '__main__': p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('--model-dir', default=str(ROOT)) p.add_argument('--runs', type=int, default=60) p.add_argument('--warmup', type=int, default=20) p.add_argument('--rounds', type=int, default=3) p.add_argument('--out', default=str(ROOT / 'benchmarks' / 'native_mlx_report.json')) p.add_argument('--worker', help=argparse.SUPPRESS) a = p.parse_args() if a.worker == 'scaling': scaling_worker(a) elif a.worker == 'parity': parity_worker(a) elif a.worker: worker(a) else: driver(a)