"""Image input shared by both NeoJev backends: decoding, resizing, Qwen patching and M-RoPE positions. Pure numpy + Pillow, so the MLX and CUDA deciders feed the vision tower exactly the same pixels. The patch layout and the position rule follow the Qwen2-VL / Qwen3-VL processor and get_rope_index. """ import base64 import binascii import io import numpy as np from PIL import Image PATCH, TEMPORAL, MERGE = 16, 2, 2 FACTOR = PATCH * MERGE # both sides must be multiples of this MIN_SIDE = FACTOR MAX_SIDE_LIMIT = 2048 MEAN = STD = 0.5 def decode_image(data): """bytes (PNG/JPEG...), base64 str, PIL image or HxWxC uint8 array -> RGB PIL image. Raises ValueError.""" if isinstance(data, Image.Image): img = data elif isinstance(data, np.ndarray): img = Image.fromarray(data) else: if isinstance(data, str): s = "".join((data.split(",", 1)[1] if data.startswith("data:") else data).split()) # allow wrapped lines try: data = base64.b64decode(s, validate=True) except (binascii.Error, ValueError) as e: raise ValueError(f"image is not valid base64: {e}") from None try: img = Image.open(io.BytesIO(data)) img.load() except Exception as e: raise ValueError(f"cannot decode image: {e}") from None if img.mode in ("RGBA", "LA", "P", "PA"): img = img.convert("RGBA") bg = Image.new("RGBA", img.size, (255, 255, 255, 255)) return Image.alpha_composite(bg, img).convert("RGB") if img.mode.startswith("I;16") or img.mode == "I": a = np.asarray(img, dtype=np.float32) img = Image.fromarray((a / max(a.max(), 1.0) * 255).astype(np.uint8)) return img.convert("RGB") def target_size(w, h, max_side): """Keep the aspect ratio, make the long side at most max_side, both sides multiples of 32 and >= 32.""" s = min(1.0, max_side / max(w, h)) tw = max(MIN_SIDE, int(round(w * s / FACTOR)) * FACTOR) th = max(MIN_SIDE, int(round(h * s / FACTOR)) * FACTOR) while max(tw, th) > max_side and max(tw, th) > MIN_SIDE: if tw >= th: tw -= FACTOR else: th -= FACTOR return tw, th def prepare_image(img, max_side=640): """-> (patches float32 [t*h*w, 3*2*16*16], grid_thw (1, h, w) in patch units).""" img = decode_image(img) tw, th = target_size(img.width, img.height, max_side) if (tw, th) != img.size: img = img.resize((tw, th), Image.BICUBIC) x = (np.asarray(img, dtype=np.float32) / 255.0 - MEAN) / STD # H, W, C x = np.repeat(x.transpose(2, 0, 1)[None], TEMPORAL, axis=0) # T, C, H, W gh, gw = th // PATCH, tw // PATCH x = x.reshape(1, TEMPORAL, 3, gh // MERGE, MERGE, PATCH, gw // MERGE, MERGE, PATCH) x = x.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8) return np.ascontiguousarray(x.reshape(gh * gw, 3 * TEMPORAL * PATCH * PATCH)), (1, gh, gw) def n_image_tokens(grid_thw): t, h, w = grid_thw return t * h * w // (MERGE * MERGE) def image_block(tok, grid_thw): """Token ids of one image in the prompt: <|vision_start|> <|image_pad|> * n <|vision_end|>.""" ids = tok.convert_tokens_to_ids(["<|vision_start|>", "<|image_pad|>", "<|vision_end|>"]) return [ids[0]] + [ids[1]] * n_image_tokens(grid_thw) + [ids[2]] def mrope_positions(ids, start, grid_thw, image_token_id): """3 x len(ids) int32 positions (temporal, height, width) for a chunk that begins at position `start`. Text tokens share one position on all three axes. An image starting at s gets (s, s + row, s + col) over the merged grid, and the next token continues at s + max(rows, cols). At most one image per chunk. """ n = len(ids) pos = np.empty((3, n), dtype=np.int32) p, i = start, 0 while i < n: if ids[i] == image_token_id and grid_thw is not None: _, gh, gw = grid_thw rows, cols = gh // MERGE, gw // MERGE k = rows * cols pos[0, i:i + k] = p pos[1, i:i + k] = p + np.repeat(np.arange(rows), cols) pos[2, i:i + k] = p + np.tile(np.arange(cols), rows) p += max(rows, cols) i += k else: pos[:, i] = p p += 1 i += 1 return pos def noul_reminder(questions): """Short restatement of the yes/no questions, placed after the image when the questions are cached in front. With the question block before a long image the model tends to lose yes/no statements; one line each after the image restores them. Covers noul questions and two-option choices (server.py sends a TypeSafe noul that way). """ lines = [] for i, (name, spec) in enumerate(questions.items(), 1): if spec["type"] == "noul": lines.append(f"Q{i} ({name}): \"{spec['statement']}\" Yes / No") elif spec["type"] == "choice" and len(spec["options"]) == 2: lines.append(f"Q{i} ({name}): \"{spec.get('prompt', '')}\" {spec['options'][0]} / {spec['options'][1]}") return "\n\nREMINDER:\n" + "\n".join(lines) if lines else ""