Hugging Face's logo

Buckets:
EmergentTwilight
/
MMCoT

Files
xet
EmergentTwilight/MMCoT / projects /VLMEvalKit_Thinkmorph /tests /test_uniar_adapter.py
EmergentTwilight's picture
EmergentTwilight
download
raw
5.33 kB
from pathlib import Path
import pytest
from vlmeval.dataset.utils.multiple_choice import extract_answer_from_item
from vlmeval.utils.matching_util import can_infer
from vlmeval.vlm.uniar_adapter import (
_cuda_device_index,
_parse_message,
_user_content,
)
def test_parse_message_preserves_interleaving() -> None:
images, ordered = _parse_message(
[
{"type": "text", "value": "compare"},
{"type": "image", "value": "/tmp/a.png"},
{"type": "text", "value": "choose A-D"},
{"type": "image", "value": "/tmp/b.png"},
]
)
assert images == [Path("/tmp/a.png"), Path("/tmp/b.png")]
assert ordered == [
("text", "compare"),
("image", 0),
("text", "choose A-D"),
("image", 1),
]
def test_user_content_is_exactly_the_framework_message() -> None:
images = [object(), object()]
ordered = [
("image", 0),
("text", "framework-owned question and options"),
("image", 1),
]
assert _user_content(ordered, images) == [
{"type": "image", "image": images[0]},
{"type": "text", "text": "framework-owned question and options"},
{"type": "image", "image": images[1]},
]
@pytest.mark.parametrize(
"message",
[
[{"type": "text", "value": "question only"}],
[{"type": "image", "value": "/tmp/a.png"}],
[{"type": "video", "value": "/tmp/a.mp4"}],
],
)
def test_parse_message_rejects_incomplete_or_unsupported_input(message) -> None:
with pytest.raises(ValueError):
_parse_message(message)
def test_cuda_device_index_after_vlmeval_rank_isolation() -> None:
assert _cuda_device_index(local_rank=3, visible_devices=1) == 0
assert _cuda_device_index(local_rank=2, visible_devices=4) == 2
with pytest.raises(RuntimeError):
_cuda_device_index(local_rank=4, visible_devices=4)
def test_exact_match_accepts_uniar_final_answer_contract() -> None:
choices = {"A": "the second image", "B": "the third image"}
assert can_infer("Final answer: B", choices) == "B"
assert can_infer("Final answer: B<|im_end|>", choices) == "B"
@pytest.mark.parametrize(
("prediction", "expected"),
[
(
"A. rubber\nB. cotton\nC. kevlar\nD. leather\n\nFinal answer: A",
"A",
),
(
"A. second\nB. third\n\nThe correct answer is **B. the third image**.",
"B",
),
(
(
"A. red, B. black, C. white, D. yellow.\n"
"Therefore, the correct option is B."
),
"B",
),
(
"A. red\nB. blue\nC. green\nD. white\n\nD. white",
"D",
),
("Initial answer: A\nFinal answer: B", "B"),
("Final answer: b<|im_end|>", "B"),
],
)
def test_exact_match_prefers_explicit_terminal_answer(
prediction: str, expected: str
) -> None:
choices = {"A": "red", "B": "blue", "C": "green", "D": "white"}
assert can_infer(prediction, choices) == expected
@pytest.mark.parametrize(
"prediction",
[
"A. red\nB. blue\nC. green\nD. white",
'I will inspect the image.\n<tool_call>{"image_id": "Picture 1"}</tool_call>',
"I considered option A.\nI still need to inspect the image.",
],
)
def test_exact_match_does_not_invent_a_terminal_answer(prediction: str) -> None:
choices = {"A": "red", "B": "blue", "C": "green", "D": "white"}
assert can_infer(prediction, choices) is False
def test_exact_text_match_does_not_mutate_choices() -> None:
choices = {"A": "Red", "B": "Blue"}
assert can_infer("red", choices) == "A"
assert choices == {"A": "Red", "B": "Blue"}
@pytest.mark.parametrize(
("prediction", "choices", "expected"),
[
("Final answer: right<|im_end|>", {"A": "right", "B": "left"}, "A"),
("The correct answer is **green**.", {"A": "green", "B": "red"}, "A"),
(
"Final answer: dark red",
{"A": "red", "B": "dark red"},
"B",
),
],
)
def test_exact_match_accepts_exact_terminal_choice_text(
prediction: str, choices: dict[str, str], expected: str
) -> None:
assert can_infer(prediction, choices) == expected
@pytest.mark.parametrize(
("prediction", "choices"),
[
("Final answer: beige", {"A": "white", "B": "black"}),
("Final answer: bright red", {"A": "red", "B": "blue"}),
("Final answer: left", {"A": "left", "B": "left"}),
],
)
def test_exact_match_rejects_nonunique_or_nonchoice_terminal_text(
prediction: str, choices: dict[str, str]
) -> None:
assert can_infer(prediction, choices) is False
def test_structured_answer_takes_priority_over_rationale() -> None:
item = {
"question": "choose one",
"prediction": "I initially considered A.\n<answer>B</answer>",
"A": "first",
"B": "second",
}
assert extract_answer_from_item(None, item)["opt"] == "B"
def test_last_complete_structured_answer_wins() -> None:
item = {
"question": "choose one",
"prediction": "<answer>A</answer>\nCorrection: <answer>B</answer>",
"A": "first",
"B": "second",
}
assert extract_answer_from_item(None, item)["opt"] == "B"

Xet Storage Details

Size:
5.33 kB
·
Xet hash:
4d3d4cffc39872227c86d7ca8b2649533cb40425e933cef455f3189aa33cc371

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.