File size: 2,396 Bytes
16c436e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
"""Run an exported d1 .aimodel from Python on a chosen compute unit (for the parity gate; the speed numbers come from
the Swift worker).

    .venv/bin/python coreai_py.py out/d1_gpu_fp16.aimodel gpu [image]
"""
import asyncio, sys, time
import numpy as np
from coreai.runtime import AIModel, ComputeUnitKind, NDArray, SpecializationOptions

KINDS = {"gpu": ComputeUnitKind.gpu, "ane": ComputeUnitKind.neural_engine, "cpu": ComputeUnitKind.cpu}


class Runner:
    def __init__(self, path, compute):
        t = time.perf_counter()
        opts = SpecializationOptions.from_preferred_compute_unit_kind(KINDS[compute]())
        self.model = asyncio.run(AIModel.load(path, opts))
        self.fn = self.model.load_function("main")
        self.load_s = time.perf_counter() - t

    def __call__(self, image):
        out = asyncio.run(self.fn({"image": NDArray(np.ascontiguousarray(image, dtype=np.float32))}))
        o = np.array(out["logits"].numpy(), dtype=np.float32)
        return o[0, :, 0, :].T if o.ndim == 4 else o  # BC1S [1, U, 1, 12] -> [12, U]


if __name__ == "__main__":
    from PIL import Image
    from mlx_vlm import load
    from d1_common import frame, pixels, mlx_logits, readout, load_meta
    from questions import QUESTIONS
    path, compute = sys.argv[1], sys.argv[2]
    r = Runner(path, compute)
    print("load+specialize %.1fs" % r.load_s)
    model, _ = load("LiquidAI/d1-2B-MLX-8bit")
    eng, meta = model.engine, load_meta()
    img = frame(Image.open(sys.argv[3] if len(sys.argv) > 3 else "../screenshot.png"))
    x = pixels(eng, img)
    t = time.perf_counter(); out = r(x); print("first run %.3fs" % (time.perf_counter() - t))
    ts = []
    for _ in range(5):
        t = time.perf_counter(); out = r(x); ts.append(time.perf_counter() - t)
    print("warm runs (python) ms:", [round(1000 * v, 1) for v in ts])
    ref = mlx_logits(eng, img, meta)
    err = out - ref
    print("logits PSNR vs MLX-8bit: %.1f dB, max abs %.3f, finite %s" % (
        10 * np.log10(np.abs(ref).max() ** 2 / np.mean(err ** 2)), np.abs(err).max(), np.isfinite(out).all()))
    a, b = readout(eng, QUESTIONS, ref, meta), readout(eng, QUESTIONS, out, meta)
    for k in QUESTIONS:
        f = (lambda d: (d["choice"], round(d["confidence"], 3))) if a[k]["type"] == "choice" else (lambda d: (d["box"], round(d["found"], 3)))
        print(k, "MLX", f(a[k]), "| coreai", f(b[k]))