neojev-mlx / vision_common.py
dowands's picture
Add files using upload-large-folder tool
74630d5 verified
Raw History Blame Contribute Delete
5.15 kB
"""Image input shared by both NeoJev backends: decoding, resizing, Qwen patching and M-RoPE positions.
Pure numpy + Pillow, so the MLX and CUDA deciders feed the vision tower exactly the same pixels.
The patch layout and the position rule follow the Qwen2-VL / Qwen3-VL processor and get_rope_index.
"""
import base64
import binascii
import io
import numpy as np
from PIL import Image
PATCH, TEMPORAL, MERGE = 16, 2, 2
FACTOR = PATCH * MERGE # both sides must be multiples of this
MIN_SIDE = FACTOR
MAX_SIDE_LIMIT = 2048
MEAN = STD = 0.5
def decode_image(data):
"""bytes (PNG/JPEG...), base64 str, PIL image or HxWxC uint8 array -> RGB PIL image. Raises ValueError."""
if isinstance(data, Image.Image):
img = data
elif isinstance(data, np.ndarray):
img = Image.fromarray(data)
else:
if isinstance(data, str):
s = "".join((data.split(",", 1)[1] if data.startswith("data:") else data).split()) # allow wrapped lines
try:
data = base64.b64decode(s, validate=True)
except (binascii.Error, ValueError) as e:
raise ValueError(f"image is not valid base64: {e}") from None
try:
img = Image.open(io.BytesIO(data))
img.load()
except Exception as e:
raise ValueError(f"cannot decode image: {e}") from None
if img.mode in ("RGBA", "LA", "P", "PA"):
img = img.convert("RGBA")
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
return Image.alpha_composite(bg, img).convert("RGB")
if img.mode.startswith("I;16") or img.mode == "I":
a = np.asarray(img, dtype=np.float32)
img = Image.fromarray((a / max(a.max(), 1.0) * 255).astype(np.uint8))
return img.convert("RGB")
def target_size(w, h, max_side):
"""Keep the aspect ratio, make the long side at most max_side, both sides multiples of 32 and >= 32."""
s = min(1.0, max_side / max(w, h))
tw = max(MIN_SIDE, int(round(w * s / FACTOR)) * FACTOR)
th = max(MIN_SIDE, int(round(h * s / FACTOR)) * FACTOR)
while max(tw, th) > max_side and max(tw, th) > MIN_SIDE:
if tw >= th:
tw -= FACTOR
else:
th -= FACTOR
return tw, th
def prepare_image(img, max_side=640):
"""-> (patches float32 [t*h*w, 3*2*16*16], grid_thw (1, h, w) in patch units)."""
img = decode_image(img)
tw, th = target_size(img.width, img.height, max_side)
if (tw, th) != img.size:
img = img.resize((tw, th), Image.BICUBIC)
x = (np.asarray(img, dtype=np.float32) / 255.0 - MEAN) / STD # H, W, C
x = np.repeat(x.transpose(2, 0, 1)[None], TEMPORAL, axis=0) # T, C, H, W
gh, gw = th // PATCH, tw // PATCH
x = x.reshape(1, TEMPORAL, 3, gh // MERGE, MERGE, PATCH, gw // MERGE, MERGE, PATCH)
x = x.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
return np.ascontiguousarray(x.reshape(gh * gw, 3 * TEMPORAL * PATCH * PATCH)), (1, gh, gw)
def n_image_tokens(grid_thw):
t, h, w = grid_thw
return t * h * w // (MERGE * MERGE)
def image_block(tok, grid_thw):
"""Token ids of one image in the prompt: <|vision_start|> <|image_pad|> * n <|vision_end|>."""
ids = tok.convert_tokens_to_ids(["<|vision_start|>", "<|image_pad|>", "<|vision_end|>"])
return [ids[0]] + [ids[1]] * n_image_tokens(grid_thw) + [ids[2]]
def mrope_positions(ids, start, grid_thw, image_token_id):
"""3 x len(ids) int32 positions (temporal, height, width) for a chunk that begins at position `start`.
Text tokens share one position on all three axes. An image starting at s gets (s, s + row, s + col) over
the merged grid, and the next token continues at s + max(rows, cols). At most one image per chunk.
"""
n = len(ids)
pos = np.empty((3, n), dtype=np.int32)
p, i = start, 0
while i < n:
if ids[i] == image_token_id and grid_thw is not None:
_, gh, gw = grid_thw
rows, cols = gh // MERGE, gw // MERGE
k = rows * cols
pos[0, i:i + k] = p
pos[1, i:i + k] = p + np.repeat(np.arange(rows), cols)
pos[2, i:i + k] = p + np.tile(np.arange(cols), rows)
p += max(rows, cols)
i += k
else:
pos[:, i] = p
p += 1
i += 1
return pos
def noul_reminder(questions):
"""Short restatement of the yes/no questions, placed after the image when the questions are cached in front.
With the question block before a long image the model tends to lose yes/no statements; one line each after
the image restores them. Covers noul questions and two-option choices (server.py sends a TypeSafe noul that way).
"""
lines = []
for i, (name, spec) in enumerate(questions.items(), 1):
if spec["type"] == "noul":
lines.append(f"Q{i} ({name}): \"{spec['statement']}\" Yes / No")
elif spec["type"] == "choice" and len(spec["options"]) == 2:
lines.append(f"Q{i} ({name}): \"{spec.get('prompt', '')}\" {spec['options'][0]} / {spec['options'][1]}")
return "\n\nREMINDER:\n" + "\n".join(lines) if lines else ""