Add MiniCPM-V 4.6 vision call (screenshot -> observation)

Implement Call 1 of the pipeline (docs/plans/04):

- prompts.py: VISION_PROMPT (perception only; bans guessing browser/OS/device/
role/environment/backend) and vision_user_text() to append the tester note.
- vision.py: build_vision_messages() (PIL image or URL), observe() running
apply_chat_template + generate + decode with downsample_mode="16x" and
max_slice_nums=36, and load_model() that imports transformers lazily so the
module stays importable on the CPU frontend (the model runs on Modal).
- tests: prompt composition, message shape, and observe() orchestration via
fakes (no model download). Verified the module imports without transformers.

https://claude.ai/code/session_01JE4YVLBWnk188Yu5sib59y

buglens/prompts.py

@@ -1,2 +1,45 @@
-"""Prompt templates for screenshot observation and report structuring."""
+"""Versioned prompts for the BugLens model pipeline.
 
+Kept in one place so prompt changes are reviewable and the vision/structuring
+calls never drift from the behavior described in docs/plans/04.
+"""
+
+from __future__ import annotations
+
+# Call 1 — perception only. The model must describe what is visible and must
+# never guess context it cannot see. This honesty is the whole product.
+VISION_PROMPT = """You are a QA assistant inspecting a UI screenshot.
+
+Describe only what is visible in the screenshot and what the tester note explicitly says.
+
+Include:
+- screen or page type if visible
+- visible buttons and whether they appear enabled, disabled, loading, or selected
+- visible error text, empty states, warnings, labels, amounts, or table states
+- visible layout issues, overlap, truncation, missing content, or broken states
+- the tester's note as user-provided context
+
+Do not infer causes.
+Do not guess browser, OS, device, user role, environment, backend state, API status, account status, or network state.
+If something is ambiguous, say it is ambiguous.
+
+Respond using exactly these sections:
+Visible facts:
+- ...
+
+Tester note:
+- ...
+
+Ambiguities:
+- ...
+
+Not visible:
+- ..."""
+
+
+def vision_user_text(note: str | None) -> str:
+    """Compose the vision instructions plus the tester note (or a clear absence)."""
+
+    note = (note or "").strip()
+    suffix = note if note else "(none provided)"
+    return f"{VISION_PROMPT}\n\nTester note: {suffix}"

buglens/vision.py

@@ -1,2 +1,90 @@
-"""Screenshot observation model call."""
+"""Call 1 of the pipeline: screenshot -> factual UI observation.
 
+`transformers` is imported lazily inside `load_model`, so this module stays
+importable on the lightweight Gradio frontend (which has no ML deps); the model
+itself is loaded and run on the Modal GPU backend. `observe()` takes the model
+and processor as arguments so the orchestration is testable without downloading
+the ~1.3B model.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from buglens.config import DEFAULT_MODEL_ID
+from buglens.prompts import vision_user_text
+
+DEFAULT_DOWNSAMPLE = "16x"  # "4x" only for tiny OCR-heavy text (slower)
+DEFAULT_MAX_NEW_TOKENS = 512
+DEFAULT_MAX_SLICE_NUMS = 36  # handles high-resolution screenshots
+
+
+def build_vision_messages(image: Any, note: str | None = None) -> list[dict]:
+    """Build the chat messages for the vision call.
+
+    `image` may be a PIL.Image (an uploaded screenshot) or a URL/path string.
+    """
+
+    if isinstance(image, str):
+        image_part = {"type": "image", "url": image}
+    else:
+        image_part = {"type": "image", "image": image}
+    return [
+        {
+            "role": "user",
+            "content": [image_part, {"type": "text", "text": vision_user_text(note)}],
+        }
+    ]
+
+
+def observe(
+    model: Any,
+    processor: Any,
+    image: Any,
+    note: str | None = None,
+    *,
+    downsample_mode: str = DEFAULT_DOWNSAMPLE,
+    max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS,
+    max_slice_nums: int = DEFAULT_MAX_SLICE_NUMS,
+) -> str:
+    """Run MiniCPM-V 4.6 to produce a factual observation of the screenshot.
+
+    `downsample_mode` must be passed to both `apply_chat_template` and `generate`
+    per the model card.
+    """
+
+    messages = build_vision_messages(image, note)
+    inputs = processor.apply_chat_template(
+        messages,
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt",
+        downsample_mode=downsample_mode,
+        max_slice_nums=max_slice_nums,
+    ).to(model.device)
+    generated = model.generate(
+        **inputs,
+        downsample_mode=downsample_mode,
+        max_new_tokens=max_new_tokens,
+    )
+    prompt_len = inputs["input_ids"].shape[-1]
+    text = processor.decode(generated[0][prompt_len:], skip_special_tokens=True)
+    return text.strip()
+
+
+def load_model(model_id: str = DEFAULT_MODEL_ID):
+    """Load MiniCPM-V 4.6 and its processor (imports transformers lazily).
+
+    Install with ``transformers[torch]>=5.7.0 torchvision av`` — ``av`` instead
+    of ``torchcodec`` avoids the CUDA-version conflict noted on the model card.
+    Called by the Modal GPU backend, not by the CPU frontend.
+    """
+
+    from transformers import AutoModelForImageTextToText, AutoProcessor  # noqa: PLC0415
+
+    processor = AutoProcessor.from_pretrained(model_id)
+    model = AutoModelForImageTextToText.from_pretrained(
+        model_id, torch_dtype="auto", device_map="auto"
+    )
+    return model, processor

pyproject.toml

@@ -36,5 +36,6 @@ target-version = "py311"
 select = ["E", "F", "I", "UP", "B"]
 
 [tool.ruff.lint.per-file-ignores]
-# Verbatim example copy reads better unwrapped than split across lines.
+# Verbatim example/prompt copy reads better unwrapped than split across lines.
 "buglens/examples.py" = ["E501"]
+"buglens/prompts.py" = ["E501"]

tests/test_vision.py

@@ -0,0 +1,70 @@
+"""Tests for the vision call orchestration (no real model download)."""
+
+from buglens import vision
+from buglens.prompts import VISION_PROMPT, vision_user_text
+
+
+def test_vision_user_text_includes_prompt_and_note() -> None:
+    text = vision_user_text("Pay button does nothing")
+    assert VISION_PROMPT in text
+    assert text.endswith("Tester note: Pay button does nothing")
+
+
+def test_vision_user_text_handles_missing_note() -> None:
+    assert vision_user_text(None).endswith("Tester note: (none provided)")
+    assert vision_user_text("   ").endswith("Tester note: (none provided)")
+
+
+def test_build_messages_url_vs_image() -> None:
+    url_msg = vision.build_vision_messages("https://x/y.png", "note")
+    assert url_msg[0]["content"][0] == {"type": "image", "url": "https://x/y.png"}
+
+    sentinel = object()  # stands in for a PIL.Image
+    img_msg = vision.build_vision_messages(sentinel)
+    assert img_msg[0]["content"][0] == {"type": "image", "image": sentinel}
+    # The instruction text always bans guessing.
+    assert "Do not guess browser" in img_msg[0]["content"][1]["text"]
+
+
+class _FakeTensor:
+    def __init__(self, length: int) -> None:
+        self.shape = (1, length)
+
+
+class _FakeInputs(dict):
+    def to(self, _device):
+        return self
+
+
+class _FakeProcessor:
+    def apply_chat_template(self, messages, **kwargs):
+        self.template_kwargs = kwargs
+        self.messages = messages
+        return _FakeInputs(input_ids=_FakeTensor(3))
+
+    def decode(self, ids, **kwargs):
+        self.decoded_ids = ids
+        return "  Visible facts:\n- A disabled Pay button  "
+
+
+class _FakeModel:
+    device = "cpu"
+
+    def generate(self, **kwargs):
+        self.generate_kwargs = kwargs
+        return [[10, 11, 12, 13, 14, 15]]
+
+
+def test_observe_passes_params_and_strips_output() -> None:
+    model, processor = _FakeModel(), _FakeProcessor()
+
+    out = vision.observe(model, processor, "shot.png", "note", downsample_mode="16x")
+
+    assert out == "Visible facts:\n- A disabled Pay button"
+    # downsample_mode + slice nums reach the template; downsample + tokens reach generate.
+    assert processor.template_kwargs["downsample_mode"] == "16x"
+    assert processor.template_kwargs["max_slice_nums"] == 36
+    assert model.generate_kwargs["downsample_mode"] == "16x"
+    assert model.generate_kwargs["max_new_tokens"] == 512
+    # Only the newly generated tokens (after the prompt length of 3) are decoded.
+    assert processor.decoded_ids == [13, 14, 15]