File size: 4,276 Bytes
9bd7482 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """System-1 multimodal decision model tests (tiny backend, CPU-only)."""
import sys
from pathlib import Path
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
torch = pytest.importorskip("torch")
from qwenjev.candidates import CandidateAction # noqa: E402
from qwenjev.system1 import QwenJevModel, System1Config # noqa: E402
from qwenjev.vision import TinyVLMConfig # noqa: E402
GRID = [
[1, 1, 1, 1, 1],
[1, 2, 0, 3, 1],
[1, 0, 4, 0, 1],
[1, 1, 1, 1, 1],
]
CANDS = [
CandidateAction("ACTION1", text="move up"),
CandidateAction("ACTION3", text="move left"),
CandidateAction("ACTION6(2,2)", x=2, y=2, text="click (2,2)"),
]
@pytest.fixture(scope="module")
def model():
cfg = System1Config(
backend="tiny",
tiny=TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64,
vision_dim=48, max_text_len=256),
)
return QwenJevModel(cfg).build()
def test_choice_probabilities_normalized(model):
d = model.choose("qwenjev-state-v1|H=4 W=5 L=0 S=1", CANDS, grid=GRID)
assert len(d.probs) == len(CANDS)
assert abs(sum(d.probs) - 1.0) < 1e-5
assert all(p >= 0 for p in d.probs)
assert d.chosen in CANDS
def test_choice_requires_two_candidates(model):
with pytest.raises(ValueError):
model.choose("state", CANDS[:1], grid=GRID)
def test_vision_changes_choice_scores(model):
"""Regression: the frame image MUST influence the decision scores."""
torch.manual_seed(0)
state = "qwenjev-state-v1|H=4 W=5 L=0 S=1"
g1 = [[v for v in row] for row in GRID]
g2 = [[v for v in row] for row in GRID]
g2[1][2] = 9 # perturb one cell
leaves1 = model.encode_question(state, CANDS, grid=g1)
leaves2 = model.encode_question(state, CANDS, grid=g2)
p1 = model.forward_choice(leaves1)
p2 = model.forward_choice(leaves2)
assert max(abs(a - b) for a, b in zip(p1, p2)) > 1e-6, (
"vision tower has no effect on Choice scores"
)
def test_vision_off_ablation(model):
"""use_vision=False still works (ablation mode) and stays finite."""
old = model.config.use_vision
try:
model.config.use_vision = False
leaves = model.encode_question("state", CANDS, grid=None)
for leaf in leaves:
assert leaf.pixel_values is None
probs = model.forward_choice(leaves)
assert abs(sum(probs) - 1.0) < 1e-5
finally:
model.config.use_vision = old
def test_boolean_and_score_heads(model):
state = "qwenjev-state-v1|H=4 W=5 L=0 S=1"
p = model.forward_boolean(state, "inst", "ACTION3 changes the frame", GRID)
assert 0.0 <= p <= 1.0
s = model.forward_score(state, "inst", "promise of state", GRID)
assert 2.0 <= s <= 10.0
def test_boolean_logits_are_trainable(model):
"""Trainable twin must keep the autograd graph (old bug: detached)."""
logits = model.forward_boolean_logits("state", "inst", "prop", GRID)
assert logits.requires_grad
logits.sum().backward() # no "does not require grad" error
def test_score_logits_are_trainable(model):
logits = model.forward_score_logits("state", "inst", "desc", GRID)
assert logits.requires_grad
def test_save_and_load_heads(model, tmp_path):
out = tmp_path / "ckpt"
model.save(str(out))
assert (out / "qwenjev_s1.pt").exists()
cfg = System1Config(
backend="tiny",
tiny=TinyVLMConfig(image_size=64, patch_size=8, lm_dim=64,
vision_dim=48, max_text_len=256),
)
m2 = QwenJevModel(cfg).build()
m2.load_heads(str(out))
model.eval(), m2.eval() # deterministic comparison
state = "qwenjev-state-v1|H=4 W=5 L=0 S=1"
p1 = model.forward_choice(model.encode_question(state, CANDS, grid=GRID))
p2 = m2.forward_choice(m2.encode_question(state, CANDS, grid=GRID))
assert max(abs(a - b) for a, b in zip(p1, p2)) < 1e-5
def test_hf_backend_config_smoke():
"""hf backend is the real-model path; verify config plumbing only
(no weights download in CI)."""
cfg = System1Config(backend="hf", hf_model="Qwen/Qwen3-VL-2B-Instruct")
assert cfg.hf_model == "Qwen/Qwen3-VL-2B-Instruct"
assert cfg.freeze_vision_tower is False
|