File size: 3,922 Bytes
9bd7482
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
"""Vision tower + TinyVLM tests (CPU-only, no downloads)."""

import sys
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

torch = pytest.importorskip("torch")

from qwenjev.vision import (  # noqa: E402
    BOS_ID,
    EOT_ID,
    PAD_ID,
    TinyVLM,
    TinyVLMConfig,
    grid_to_tensor,
    render_grid_pil,
    text_to_byte_ids,
)


GRID = [
    [1, 1, 1, 1, 1],
    [1, 2, 0, 3, 1],
    [1, 0, 4, 0, 1],
    [1, 1, 1, 1, 1],
]


def test_text_to_byte_ids_roundtrip():
    text = "STATE:\nqwenjev-state-v1"
    ids = text_to_byte_ids(text, max_len=512)
    assert ids[0] == BOS_ID and ids[-1] == EOT_ID
    body = bytes(ids[1:-1]).decode("utf-8")
    assert body == text


def test_text_truncation():
    ids = text_to_byte_ids("x" * 999, max_len=64)
    assert len(ids) == 64 and ids[-1] == EOT_ID


def test_grid_to_tensor_values():
    t = grid_to_tensor(GRID, image_size=64)
    assert t.shape == (3, 64, 64)
    assert 0.0 <= t.min().item() and t.max().item() <= 1.0
    # a coloured cell maps to a non-black pixel somewhere in the image
    assert t.sum().item() > 0


def test_grid_to_tensor_distinguishes_values():
    g1 = [[0, 0], [0, 0]]
    g2 = [[0, 1], [0, 0]]
    t1, t2 = grid_to_tensor(g1, 32), grid_to_tensor(g2, 32)
    assert not torch.allclose(t1, t2)


def test_render_grid_pil():
    pil = pytest.importorskip("PIL")
    img = render_grid_pil(GRID, cell_px=4)
    assert isinstance(img, pil.Image.Image)
    assert img.size == (5 * 4, 4 * 4)


def test_tinyvlm_forward_shapes():
    m = TinyVLM(TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64))
    px = torch.stack([grid_to_tensor(GRID, 64), grid_to_tensor(GRID, 64)])
    ids = torch.tensor([
        text_to_byte_ids("abcd", 32),
        text_to_byte_ids("defg", 32),
    ])
    out, hs = m(px, ids, output_hidden_states=True)
    n_vis = m.n_vision_tokens
    assert out.shape == (2, n_vis + ids.shape[1], 64)
    # hidden states: [embed, after_layer1, after_layer2]
    assert len(hs) == 3
    assert all(h.shape == out.shape for h in hs)


def test_tinyvlm_all_finite_with_padding():
    """Padding + causal masks must never produce NaN/Inf (regression)."""
    torch.manual_seed(0)
    m = TinyVLM(TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64))
    px = torch.stack([grid_to_tensor(GRID, 64)])
    long_ids = text_to_byte_ids("a long state text " * 3, 128)
    short_ids = text_to_byte_ids("tiny", 128)
    ids = torch.tensor([long_ids, short_ids + [PAD_ID] * (len(long_ids) - len(short_ids))])
    attn = torch.tensor([
        [1] * len(long_ids),
        [1] * len(short_ids) + [0] * (len(long_ids) - len(short_ids)),
    ])
    out, hs = m(px.expand(2, -1, -1, -1), ids, attention_mask=attn,
                output_hidden_states=True)
    for h in hs:
        assert torch.isfinite(h).all(), "non-finite hidden states (mask bug)"
    assert torch.isfinite(out).all()


def test_tinyvlm_pooled_uses_last_text_position():
    m = TinyVLM(TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64))
    px = torch.stack([grid_to_tensor(GRID, 64)])
    ids = torch.tensor([text_to_byte_ids("hello", 32)])
    out, _ = m(px, ids)
    n_vis = m.n_vision_tokens
    pooled = m.pooled(out, torch.tensor([ids.shape[1]]) + n_vis)
    assert torch.allclose(pooled[0], out[0, n_vis + ids.shape[1] - 1])


def test_tinyvlm_vision_changes_representation():
    """The vision tower MUST matter: different frames -> different pooled."""
    torch.manual_seed(3)
    m = TinyVLM(TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64)).eval()
    g1 = [[0] * 8 for _ in range(8)]
    g2 = [[0] * 8 for _ in range(8)]
    g2[3][3] = 7
    ids = torch.tensor([text_to_byte_ids("state", 32)])
    with torch.no_grad():
        o1, _ = m(torch.stack([grid_to_tensor(g1, 64)]), ids)
        o2, _ = m(torch.stack([grid_to_tensor(g2, 64)]), ids)
    assert not torch.allclose(o1[0, 0], o2[0, 0])