AuroraASR-1M-MLX / scripts /bench_native.py
cowWhySo's picture
Standard layout: model.safetensors (mlx metadata), clean config.json, scripts/; reproducible tensor hash
2d44270 verified
Raw History Blame Contribute Delete
12.6 kB
#!/usr/bin/env python3
"""Native MLX benchmark: parity gate, latency variants, length scaling, memory. No ONNX Runtime.
Every latency variant runs in its own fresh process (so peak memory and RSS belong to that
variant alone) and the variants are interleaved over several rounds in shuffled order, which
spreads thermal drift and background load across them instead of favouring whichever ran first.
python scripts/bench_native.py --model-dir . --rounds 3 --runs 60 --out benchmarks/native_mlx_report.json
Apple Silicon macOS only. Cross-backend (ONNX / Core ML) numbers come from the sibling repos.
"""
import argparse
import json
import platform
import random
import resource
import subprocess
import sys
import time
from pathlib import Path
import numpy as np
ROOT = Path(__file__).resolve().parent.parent # repo root; this file lives in scripts/
# name -> (compile, output path, uses cached device input?)
VARIANTS = {
'eager_logits_host': dict(compile=False, out='logits', cached=False), # first-release path
'compiled_logits_host': dict(compile=True, out='logits', cached=False),
'eager_ids_host': dict(compile=False, out='ids', cached=False),
'compiled_ids_host': dict(compile=True, out='ids', cached=False), # recommended
'compiled_device_only': dict(compile=True, out='device', cached=True), # diagnostic: no host traffic
}
PIPELINES = {'pipeline_numpy_frontend': 'numpy', 'pipeline_mlx_frontend': 'mlx'}
def pctl(samples):
a = np.asarray(samples)
return {'p50_ms': float(np.median(a)), 'p95_ms': float(np.percentile(a, 95)),
'mean_ms': float(a.mean()), 'min_ms': float(a.min()), 'runs': int(a.size)}
def mib(n):
return n / 2**20
# ----------------------------------------------------------------------------- worker (one process)
def worker(args):
rss_import = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
import mlx.core as mx
sys.path.insert(0, str(ROOT))
from aurora_pipeline import Transcriber, read_audio
name, runs, warm = args.worker, args.runs, args.warmup
model_dir = Path(args.model_dir)
fix = np.load(model_dir / 'reference_fixture.npz')
x = np.ascontiguousarray(fix['acoustic_input'], dtype=np.float32)
expected = fix['reference_logits']
mx.reset_peak_memory()
if name in VARIANTS:
v = VARIANTS[name]
asr = Transcriber(model_dir, compile=v['compile'])
x_dev = mx.array(x)
mx.eval(x_dev)
if v['out'] == 'logits':
def once(inp=None):
return np.asarray(asr.logits(x if inp is None else inp))
elif v['out'] == 'ids':
def once(inp=None):
return asr.ids(x if inp is None else inp)
else:
def once(inp=None):
mx.eval(asr._logits(x_dev))
parity = None
if v['out'] != 'device':
y = np.asarray(asr.logits(x))
parity = {'max_abs': float(np.abs(y - expected).max()),
'argmax_agreement': float((y.argmax(-1) == expected.argmax(-1)).mean())}
for _ in range(warm):
once()
else:
asr = Transcriber(model_dir, compile=True, frontend=PIPELINES[name])
wave = read_audio(model_dir / 'reference_jfk.flac')
parity = {'transcript': asr.transcribe(wave)}
def once(inp=None):
return asr.transcribe(wave)
for _ in range(warm):
once()
times = []
for _ in range(runs):
t = time.perf_counter_ns()
once()
times.append((time.perf_counter_ns() - t) / 1e6)
out = {'name': name, 'latency': pctl(times), 'parity': parity,
'mlx_peak_memory_mib': mib(mx.get_peak_memory()),
'mlx_active_memory_mib': mib(mx.get_active_memory()),
'mlx_cache_memory_mib': mib(mx.get_cache_memory()),
'process_peak_rss_mib': mib(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss),
'rss_after_imports_only_mib': mib(rss_import)}
print(json.dumps(out))
def scaling_worker(args):
"""Latency vs clip length and batch size, compiled id path, JFK tiled to the target length."""
sys.path.insert(0, str(ROOT))
from aurora_pipeline import SAMPLE_RATE, Transcriber, read_audio
model_dir = Path(args.model_dir)
asr = Transcriber(model_dir, compile=True)
base = read_audio(model_dir / 'reference_jfk.flac')
rows = []
for seconds in (2, 5.5, 11, 22, 44, 88):
wave = np.tile(base, int(np.ceil(seconds * SAMPLE_RATE / base.size)))[:int(seconds * SAMPLE_RATE)]
feats = asr.features(wave)
for _ in range(8):
asr.ids(feats)
ts = []
for _ in range(max(10, args.runs // 3)):
t = time.perf_counter_ns()
asr.ids(feats)
ts.append((time.perf_counter_ns() - t) / 1e6)
p50 = float(np.median(ts))
rows.append({'audio_seconds': seconds, 'mel_frames': int(feats.shape[1]), 'acoustic_p50_ms': p50,
'rtf': p50 / (seconds * 1000), 'x_realtime': seconds * 1000 / p50})
batch_rows = []
f11 = np.asarray(asr.features(base)[0])
for b in (1, 4, 8, 16):
batch = np.repeat(f11[None], b, axis=0)
for _ in range(8):
asr.ids(batch)
ts = []
for _ in range(max(10, args.runs // 3)):
t = time.perf_counter_ns()
asr.ids(batch)
ts.append((time.perf_counter_ns() - t) / 1e6)
p50 = float(np.median(ts))
batch_rows.append({'batch': b, 'acoustic_p50_ms': p50, 'ms_per_utterance': p50 / b,
'utterances_per_s': b * 1000 / p50})
print(json.dumps({'length_scaling': rows, 'batch_scaling_11s': batch_rows}))
def parity_worker(args):
"""Reference-logit parity per length (torch inputs), then waveform -> logits for each frontend."""
sys.path.insert(0, str(ROOT))
from aurora_pipeline import Transcriber
model_dir = Path(args.model_dir)
fix = np.load(model_dir / 'parity_fixtures.npz')
meta = json.loads(bytes(fix['meta_json']).decode())
asr = {fe: Transcriber(model_dir, compile=True, frontend=fe) for fe in ('numpy', 'mlx')}
cases = [k for k in meta['cases'] if k != 'batch']
rows = []
for name in cases:
ref = fix[f'{name}__logits']
y = np.asarray(asr['numpy'].logits(fix[f'{name}__input']))
d = np.abs(y - ref)
row = {'case': name, 'mel_frames': meta['cases'][name]['mel_frames'], 'logit_frames': int(y.shape[1]),
'torch_features': {'max_abs': float(d.max()), 'mean_abs': float(d.mean()),
'p99_abs': float(np.percentile(d, 99)),
'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()),
'text_match': asr['numpy'].decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}}
for fe, a in asr.items(): # full path from the waveform, using this repo's own frontend
y = np.asarray(a.logits(a.features(fix[f'{name}__waveform'])))
d = np.abs(y - ref)
row[f'waveform_{fe}_frontend'] = {
'max_abs': float(d.max()), 'mean_abs': float(d.mean()),
'argmax_agreement': float((y.argmax(-1) == ref.argmax(-1)).mean()),
'text_match': a.decode_ids(y.argmax(-1)[0]) == meta['cases'][name]['text']}
rows.append(row)
x, n = fix['batch__input'], fix['batch__lengths']
ref, out_len = fix['batch__logits'], fix['batch__output_lengths']
y = np.asarray(asr['numpy'].logits(x, n))
batch = {'items': [], 'input_shape': list(x.shape)}
for i, L in enumerate(out_len):
d = np.abs(y[i, :L] - ref[i, :L])
batch['items'].append({'valid_logit_frames': int(L), 'max_abs': float(d.max()),
'argmax_agreement': float((y[i, :L].argmax(-1) == ref[i, :L].argmax(-1)).mean())})
print(json.dumps({'torch': meta['torch'], 'per_length': rows, 'padded_batch': batch}))
# ----------------------------------------------------------------------------- driver
def spawn(args, name, extra=()):
cmd = [sys.executable, '-I', str(Path(__file__).resolve()), '--model-dir', str(args.model_dir),
'--runs', str(args.runs), '--warmup', str(args.warmup), *extra, '--worker', name]
res = subprocess.run(cmd, capture_output=True, text=True)
if res.returncode:
raise RuntimeError(f'{name} failed:\n{res.stderr}')
return json.loads(res.stdout.strip().splitlines()[-1])
def driver(args):
if platform.system() != 'Darwin' or platform.machine() != 'arm64':
raise SystemExit('Run on Apple Silicon macOS.')
names = list(VARIANTS) + list(PIPELINES)
rng = random.Random(1337)
results = {n: [] for n in names}
for r in range(args.rounds):
order = names[:]
rng.shuffle(order)
for n in order:
results[n].append(spawn(args, n))
print(f'round {r + 1}/{args.rounds} done', file=sys.stderr)
summary = {}
for n, rs in results.items():
# Pool per-round medians; a round's own p50/p95 are kept so run-to-run spread is visible.
p50s = [r['latency']['p50_ms'] for r in rs]
summary[n] = {
'p50_ms_median_of_rounds': float(np.median(p50s)),
'p50_ms_per_round': p50s,
'p95_ms_median_of_rounds': float(np.median([r['latency']['p95_ms'] for r in rs])),
'runs_per_round': rs[0]['latency']['runs'], 'rounds': len(rs),
'parity': rs[0]['parity'],
'mlx_peak_memory_mib': max(r['mlx_peak_memory_mib'] for r in rs),
'mlx_active_memory_mib': max(r['mlx_active_memory_mib'] for r in rs),
'mlx_cache_memory_mib': max(r['mlx_cache_memory_mib'] for r in rs),
'process_peak_rss_mib': max(r['process_peak_rss_mib'] for r in rs),
'rss_after_imports_only_mib': min(r['rss_after_imports_only_mib'] for r in rs),
}
base = summary['eager_logits_host']['p50_ms_median_of_rounds']
for n, s in summary.items():
s['speedup_vs_eager_logits_host'] = base / s['p50_ms_median_of_rounds'] if n in VARIANTS else None
report = {
'platform': platform.platform(), 'machine': platform.machine(),
'cpu': subprocess.run(['sysctl', '-n', 'machdep.cpu.brand_string'], capture_output=True, text=True).stdout.strip(),
'python': platform.python_version(),
'input': {'mel_frames': 1101, 'audio_seconds': 11.0, 'fixture': 'reference_fixture.npz'},
'protocol': f'{args.rounds} rounds x {args.runs} timed runs after {args.warmup} warmup, each variant in a '
'fresh process, order shuffled per round',
'variants': summary,
'parity_multi_length': spawn(args, 'parity'),
'scaling': spawn(args, 'scaling'),
'notes': ['host-output variants include host-to-device upload of the 1101x80 features and device-to-host copy of the result',
'process_peak_rss_mib includes the Python/MLX runtime; rss_after_imports_only_mib is the same process before the model loads',
'mlx_* memory is the MLX allocator; mlx_cache_memory_mib is reusable freed buffers, not live tensors',
'single machine, single clip; no WER evaluation']}
import mlx.core as mx
report['mlx_version'] = getattr(mx, '__version__', None) or subprocess.run(
[sys.executable, '-c', 'import importlib.metadata as m;print(m.version("mlx"))'],
capture_output=True, text=True).stdout.strip()
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=2) + '\n')
print(json.dumps({n: (round(s['p50_ms_median_of_rounds'], 3), round(s['p95_ms_median_of_rounds'], 3),
round(s['process_peak_rss_mib'], 1)) for n, s in summary.items()}, indent=2))
print(f'saved {out}')
if __name__ == '__main__':
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('--model-dir', default=str(ROOT))
p.add_argument('--runs', type=int, default=60)
p.add_argument('--warmup', type=int, default=20)
p.add_argument('--rounds', type=int, default=3)
p.add_argument('--out', default=str(ROOT / 'benchmarks' / 'native_mlx_report.json'))
p.add_argument('--worker', help=argparse.SUPPRESS)
a = p.parse_args()
if a.worker == 'scaling':
scaling_worker(a)
elif a.worker == 'parity':
parity_worker(a)
elif a.worker:
worker(a)
else:
driver(a)