File size: 3,781 Bytes
7a102c1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Stage 3 parity gate: each Core AI variant vs MLX system_one (LiquidAI/d1-2B-MLX-8bit) on real frames.

    .venv/bin/python parity.py out/d1_gpu_fp16.aimodel:gpu out/d1_ane_fp16.aimodel:ane      # -> out/parity.md

Pass per frame: same top choice for hand and face, probability of that choice within 0.05, each box edge within 0.03
of the frame (and the same found / not found decision).
"""
import json, sys
import numpy as np
from PIL import Image, ImageOps
from mlx_vlm import load
from coreai_py import Runner
from d1_common import frame, pixels, mlx_logits, readout, load_meta
from questions import QUESTIONS

S = Image.open("../screenshot.png").convert("RGB")
I = Image.open("portrait.jpg")  # any square portrait photo (ours is not shipped).convert("RGB")
w, h = S.size
FRAMES = {
    "screenshot": S,
    "screenshot_flip": ImageOps.mirror(S),
    "screenshot_center60": S.crop((int(w * .2), int(h * .2), int(w * .8), int(h * .8))),
    "screenshot_left": S.crop((0, 0, w // 2, h)),
    "ivan": I,
    "ivan_flip": ImageOps.mirror(I),
    "ivan_top70": I.crop((0, 0, I.width, int(I.height * .7))),
    "blank_gray": Image.new("RGB", (640, 360), "gray"),
    "blank_black": Image.new("RGB", (640, 360), "black"),
}


def compare(ref, got):
    bad = []
    for k in ("hand", "face"):
        r, g = ref[k], got[k]
        if r["choice"] != g["choice"]:
            bad.append("%s %s->%s" % (k, r["choice"], g["choice"]))
        d = abs(r["probabilities"][r["choice"]] - g["probabilities"][r["choice"]])
        if d > 0.05:
            bad.append("%s p diff %.3f" % (k, d))
    edges = 0.0
    for k in ("hand_box", "face_box"):
        r, g = ref[k], got[k]
        if (r["box"] is None) != (g["box"] is None):
            bad.append("%s found %s->%s" % (k, r["box"] is not None, g["box"] is not None))
        elif r["box"] is not None:
            e = max(abs(r["box"][s] - g["box"][s]) for s in ("top", "bottom", "left", "right"))
            edges = max(edges, e)
            if e > 0.03:
                bad.append("%s edge %.3f" % (k, e))
    return bad, edges


def short(a):
    return "%s %.2f / %s %.2f" % (a["hand"]["choice"], a["hand"]["confidence"], a["face"]["choice"], a["face"]["confidence"])


model, _ = load("LiquidAI/d1-2B-MLX-8bit")
eng, meta = model.engine, load_meta()
variants = [(a.split(":")[0], a.split(":")[1]) for a in sys.argv[1:]]
runners = {(p, u): Runner(p, u) for p, u in variants}
rows, ok = [], {v: 0 for v in variants}
for name, img in FRAMES.items():
    f = frame(img)
    ref_logits = mlx_logits(eng, f, meta)
    ref = model.system_one(None, QUESTIONS, media=[f])["answers"]
    x = pixels(eng, f)
    row = ["%s" % name, short(ref)]
    for v in variants:
        out = runners[v](x)
        got = readout(eng, QUESTIONS, out, meta)
        e = out - ref_logits
        psnr = 10 * np.log10(np.abs(ref_logits).max() ** 2 / np.mean(e ** 2))
        bad, edges = compare(ref, got)
        dp = max(abs(ref[k]["probabilities"][ref[k]["choice"]] - got[k]["probabilities"][ref[k]["choice"]]) for k in ("hand", "face"))
        ok[v] += not bad
        row.append("%s | %.1f | %.3f | %.3f | %s" % (short(got), psnr, dp, edges, "PASS" if not bad else "FAIL: " + "; ".join(bad)))
    rows.append(row)
    print(" | ".join(row), flush=True)

hdr = "| frame | MLX hand / face | " + " | ".join("%s on %s: hand / face | logits PSNR dB | max Δp top | max Δedge | verdict" % (p.split("/")[-1], u) for p, u in variants) + " |"
sep = "|" + "---|" * (2 + 5 * len(variants))
md = [hdr, sep] + ["| " + " | ".join(r) + " |" for r in rows]
md.append("")
md += ["%s on %s: **%d / %d frames pass**" % (p, u, ok[(p, u)], len(FRAMES)) for p, u in variants]
open("out/parity.md", "w").write("\n".join(md) + "\n")
print("\n".join(md[-len(variants):]))