File size: 6,044 Bytes
7a102c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | """What every backend shares: the constant prompt layout for our four questions at 640x352, and the host-side
readout that turns the exported function's [12, U] logits into system_one's answers (mlx-vlm's own code).
meta = build_meta(engine) # needs the mlx-vlm engine (tokenizer, config); cached to prompt.json
answers = readout(engine, logits) # logits [12, U] as float32
"""
import json, os
import numpy as np
from PIL import Image
HERE = os.path.dirname(os.path.abspath(__file__))
META = os.path.join(HERE, "prompt.json")
W, H = 640, 360 # what index.html sends; the processor makes it 640x352
def crop16x9(w, h):
"""The centre 16:9 rectangle of a w x h picture, (left, top, right, bottom) in pixels."""
if w * 9 > h * 16:
nw = h * 16 // 9
return (w - nw) // 2, 0, (w - nw) // 2 + nw, h
if w * 9 < h * 16:
nh = w * 9 // 16
return 0, (h - nh) // 2, w, (h - nh) // 2 + nh
return 0, 0, w, h
def frame(img):
"""Any picture as the page would send it: centre-cropped to 16:9, 640x360 RGB."""
img = img.convert("RGB")
c = crop16x9(*img.size)
if c != (0, 0) + img.size:
img = img.crop(c)
return img.resize((W, H), Image.Resampling.BICUBIC)
def uncrop_box(box, w, h):
"""A box in fractions of frame()'s crop, as fractions of the original w x h picture."""
l, t, r, b = crop16x9(w, h)
return {"top": (t + box["top"] * (b - t)) / h, "bottom": (t + box["bottom"] * (b - t)) / h,
"left": (l + box["left"] * (r - l)) / w, "right": (l + box["right"] * (r - l)) / w}
def engine_parts(engine, questions):
from mlx_vlm.models.d1_lfm3_vl import d1 as D, prompt as P
qs = list(engine.named(questions).values())
return D, P, qs
def build_meta(engine, questions, write=True):
D, P, qs = engine_parts(engine, questions)
tok = engine.tokenizer
markup, parts = engine.media.encode(D.media_list([frame(Image.new("RGB", (W, H), "gray"))]))
grid = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
prefix = P.prefix_text(tok, None, engine.bos, engine.state_style, engine.system, markup)
layouts = [P.suffix_layout(tok, q, engine.lead) for q in qs]
first = tok.encode(prefix + layouts[0][0], add_special_tokens=False)
ids, state_len, question, ends = D.pack_set(tok, prefix, [t for t, _ in layouts], P.question_header(tok, None),
[o for _, o in layouts], first)
slots, tables = [], []
for j, (q, at) in enumerate(zip(qs, ends)):
for r, pos in zip(P.question_slots(q, tok), at):
tokens, forms, _ = P.readout_table(tok, r)
slots.append({"question": j, "pos": pos, "kind": type(r).__name__, "tempered": not P.typed(q)})
tables.append([[tokens[i] for i in row] for row in forms.tolist()])
union = list(dict.fromkeys(t for tab in tables for row in tab for t in row))
at = {t: i for i, t in enumerate(union)}
for s, tab in zip(slots, tables):
s["forms"] = [[at[t] for t in row] for row in tab]
image_pos = [i for i, t in enumerate(ids) if t == engine.model.config.image_token_id]
assert image_pos == list(range(image_pos[0], image_pos[0] + len(image_pos))), "image tokens not contiguous"
meta = {"ids": ids, "state_len": state_len, "question": question, "grid_thw": grid,
"image_start": image_pos[0], "image_len": len(image_pos), "union": union, "slots": slots,
"sliding_window": engine.model.config.text_config.sliding_window}
if write:
with open(META, "w") as f:
json.dump(meta, f)
return meta
def load_meta():
with open(META) as f:
return json.load(f)
def pixels(engine, img):
"""The processor's own pixel_values for a 640x360 frame, folded back into the [1, 3, 352, 640] image the exported
graph takes (exact: the patches are a permutation of it, both temporal copies equal)."""
from mlx_vlm.models.d1_lfm3_vl.media import media_list
_, parts = engine.media.encode(media_list([img]))
pv = np.array(parts[0]["pixel_values"], dtype=np.float32)
t, gh, gw = [int(v) for v in np.array(parts[0]["image_grid_thw"]).reshape(-1)]
x = pv.reshape(gh // 2, gw // 2, 2, 2, 3, 2, 16, 16)[:, :, :, :, :, 0] # (bh, bw, mr, mc, C, ph, pw)
return x.transpose(4, 0, 2, 5, 1, 3, 6).reshape(1, 3, gh * 16, gw * 16)
def mlx_logits(engine, img, meta=None):
"""The MLX reference's own [12, U] logits for a frame (the same pass system_one makes), to compare numerically."""
import mlx.core as mx
from mlx_vlm.models.d1_lfm3_vl.media import media_list
meta = meta or load_meta()
model = engine.model
_, parts = engine.media.encode(media_list([img]))
feats = [model.d1_features(p) for p in parts]
hidden = model.d1_hidden(meta["ids"], meta["state_len"], meta["question"], feats)
at = mx.array([s["pos"] for s in meta["slots"]])
logits = (hidden[at] @ model.language_model.model.embed_tokens(mx.array(meta["union"])).T).astype(mx.float32)
return np.array(logits)
def readout(engine, questions, logits, meta=None):
"""system_one's answers from the exported logits, via mlx-vlm's own probabilities / calibration / box code."""
from mlx_vlm.models.d1_lfm3_vl.prompt import Box, Boxes, Choice, probabilities, question_slots
meta = meta or load_meta()
named = engine.named(questions)
qs = list(named.values())
logits = np.asarray(logits, dtype=np.float64)
own = [[] for _ in qs]
k = 0
for j, q in enumerate(qs):
for r in question_slots(q, engine.tokenizer):
s = meta["slots"][k]
values = logits[k][np.array(s["forms"])].max(-1).tolist()
own[j].append(probabilities(r, values, engine.calibration if s["tempered"] else None))
k += 1
out = {}
for (name, q), probs in zip(named.items(), own):
p = probs if isinstance(q, (Box, Boxes)) else probs[0]
out[name] = engine.answer(None, q, p)
return out
|