File size: 5,148 Bytes
74630d5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""Image input shared by both NeoJev backends: decoding, resizing, Qwen patching and M-RoPE positions.

Pure numpy + Pillow, so the MLX and CUDA deciders feed the vision tower exactly the same pixels.
The patch layout and the position rule follow the Qwen2-VL / Qwen3-VL processor and get_rope_index.
"""
import base64
import binascii
import io

import numpy as np
from PIL import Image

PATCH, TEMPORAL, MERGE = 16, 2, 2
FACTOR = PATCH * MERGE           # both sides must be multiples of this
MIN_SIDE = FACTOR
MAX_SIDE_LIMIT = 2048
MEAN = STD = 0.5


def decode_image(data):
    """bytes (PNG/JPEG...), base64 str, PIL image or HxWxC uint8 array -> RGB PIL image. Raises ValueError."""
    if isinstance(data, Image.Image):
        img = data
    elif isinstance(data, np.ndarray):
        img = Image.fromarray(data)
    else:
        if isinstance(data, str):
            s = "".join((data.split(",", 1)[1] if data.startswith("data:") else data).split())  # allow wrapped lines
            try:
                data = base64.b64decode(s, validate=True)
            except (binascii.Error, ValueError) as e:
                raise ValueError(f"image is not valid base64: {e}") from None
        try:
            img = Image.open(io.BytesIO(data))
            img.load()
        except Exception as e:
            raise ValueError(f"cannot decode image: {e}") from None
    if img.mode in ("RGBA", "LA", "P", "PA"):
        img = img.convert("RGBA")
        bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
        return Image.alpha_composite(bg, img).convert("RGB")
    if img.mode.startswith("I;16") or img.mode == "I":
        a = np.asarray(img, dtype=np.float32)
        img = Image.fromarray((a / max(a.max(), 1.0) * 255).astype(np.uint8))
    return img.convert("RGB")


def target_size(w, h, max_side):
    """Keep the aspect ratio, make the long side at most max_side, both sides multiples of 32 and >= 32."""
    s = min(1.0, max_side / max(w, h))
    tw = max(MIN_SIDE, int(round(w * s / FACTOR)) * FACTOR)
    th = max(MIN_SIDE, int(round(h * s / FACTOR)) * FACTOR)
    while max(tw, th) > max_side and max(tw, th) > MIN_SIDE:
        if tw >= th:
            tw -= FACTOR
        else:
            th -= FACTOR
    return tw, th


def prepare_image(img, max_side=640):
    """-> (patches float32 [t*h*w, 3*2*16*16], grid_thw (1, h, w) in patch units)."""
    img = decode_image(img)
    tw, th = target_size(img.width, img.height, max_side)
    if (tw, th) != img.size:
        img = img.resize((tw, th), Image.BICUBIC)
    x = (np.asarray(img, dtype=np.float32) / 255.0 - MEAN) / STD    # H, W, C
    x = np.repeat(x.transpose(2, 0, 1)[None], TEMPORAL, axis=0)      # T, C, H, W
    gh, gw = th // PATCH, tw // PATCH
    x = x.reshape(1, TEMPORAL, 3, gh // MERGE, MERGE, PATCH, gw // MERGE, MERGE, PATCH)
    x = x.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8)
    return np.ascontiguousarray(x.reshape(gh * gw, 3 * TEMPORAL * PATCH * PATCH)), (1, gh, gw)


def n_image_tokens(grid_thw):
    t, h, w = grid_thw
    return t * h * w // (MERGE * MERGE)


def image_block(tok, grid_thw):
    """Token ids of one image in the prompt: <|vision_start|> <|image_pad|> * n <|vision_end|>."""
    ids = tok.convert_tokens_to_ids(["<|vision_start|>", "<|image_pad|>", "<|vision_end|>"])
    return [ids[0]] + [ids[1]] * n_image_tokens(grid_thw) + [ids[2]]


def mrope_positions(ids, start, grid_thw, image_token_id):
    """3 x len(ids) int32 positions (temporal, height, width) for a chunk that begins at position `start`.

    Text tokens share one position on all three axes. An image starting at s gets (s, s + row, s + col) over
    the merged grid, and the next token continues at s + max(rows, cols). At most one image per chunk.
    """
    n = len(ids)
    pos = np.empty((3, n), dtype=np.int32)
    p, i = start, 0
    while i < n:
        if ids[i] == image_token_id and grid_thw is not None:
            _, gh, gw = grid_thw
            rows, cols = gh // MERGE, gw // MERGE
            k = rows * cols
            pos[0, i:i + k] = p
            pos[1, i:i + k] = p + np.repeat(np.arange(rows), cols)
            pos[2, i:i + k] = p + np.tile(np.arange(cols), rows)
            p += max(rows, cols)
            i += k
        else:
            pos[:, i] = p
            p += 1
            i += 1
    return pos


def noul_reminder(questions):
    """Short restatement of the yes/no questions, placed after the image when the questions are cached in front.

    With the question block before a long image the model tends to lose yes/no statements; one line each after
    the image restores them. Covers noul questions and two-option choices (server.py sends a TypeSafe noul that way).
    """
    lines = []
    for i, (name, spec) in enumerate(questions.items(), 1):
        if spec["type"] == "noul":
            lines.append(f"Q{i} ({name}): \"{spec['statement']}\" Yes / No")
        elif spec["type"] == "choice" and len(spec["options"]) == 2:
            lines.append(f"Q{i} ({name}): \"{spec.get('prompt', '')}\" {spec['options'][0]} / {spec['options'][1]}")
    return "\n\nREMINDER:\n" + "\n".join(lines) if lines else ""