Instructions to use mis2/ViSS-8B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use mis2/ViSS-8B with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("image-text-to-text", model="mis2/ViSS-8B")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
pipe(text=messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoProcessor, AutoModelForMultimodalLM

processor = AutoProcessor.from_pretrained("mis2/ViSS-8B")
model = AutoModelForMultimodalLM.from_pretrained("mis2/ViSS-8B", device_map="auto")
messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"},
            {"type": "text", "text": "What animal is on the candy?"}
        ]
    },
]
inputs = processor.apply_chat_template(
	messages,
	add_generation_prompt=True,
	tokenize=True,
	return_dict=True,
	return_tensors="pt",
).to(model.device)

outputs = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:]))

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use mis2/ViSS-8B with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "mis2/ViSS-8B"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "mis2/ViSS-8B",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker

docker model run hf.co/mis2/ViSS-8B

SGLang

How to use mis2/ViSS-8B with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "mis2/ViSS-8B" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "mis2/ViSS-8B",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "mis2/ViSS-8B" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "mis2/ViSS-8B",
		"messages": [
			{
				"role": "user",
				"content": [
					{
						"type": "text",
						"text": "Describe this image in one sentence."
					},
					{
						"type": "image_url",
						"image_url": {
							"url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"
						}
					}
				]
			}
		]
	}'

Docker Model Runner
How to use mis2/ViSS-8B with Docker Model Runner:
```
docker model run hf.co/mis2/ViSS-8B
```

ViSS-8B / code /tests /test_third_view_disagreement_agent.py

mis2

Release ViSS-8B weights, processors and source code

78ac017 verified 21 days ago

Raw History Blame Contribute Delete

2.39 kB

from experiments.materialize_third_view_disagreement_agent import materialize


IDENTITY = {
    "benchmark": "demo",
    "model_manifest_sha256": "m",
    "annotation_sha256": "a",
    "evaluation_protocol_sha256": "p",
    "annotation": "annotation.json",
    "evaluation_protocol": {},
    "model": "model",
    "processor": "processor",
}


def _result(rows: list[dict]) -> dict:
    return {**IDENTITY, "rows": rows}


def test_third_view_commits_only_on_acquired_disagreement() -> None:
    recent = _result(
        [
            {"uid": "u0", "prediction": "A", "visual_tokens": 10},
            {"uid": "u1", "prediction": "D", "visual_tokens": 10},
        ]
    )
    event = _result(
        [
            {
                "uid": "u0",
                "prediction": "B",
                "error": None,
                "visual_tokens": 20,
                "sampling": {"frame_indices": list(range(20))},
            }
        ]
    )
    base = _result(
        [
            {
                "uid": "u0",
                "prediction": "A",
                "agent_policy": {"escalated": True},
                "media_calls": 2,
                "num_turns": 2,
                "model_processed_frames": 24,
                "sampling": {"frame_indices": list(range(20))},
            },
            {"uid": "u1", "prediction": "D", "media_calls": 1},
        ]
    )
    third = _result(
        [
            {
                "uid": "u0",
                "prediction": "C",
                "raw_response": "C",
                "error": None,
                "query_time": 31.0,
                "frames_observed": 32,
                "visual_tokens": 30,
                "sampling": {
                    "frame_indices": list(range(32)),
                    "frame_timestamps": list(map(float, range(32))),
                },
            }
        ]
    )
    result = materialize(recent, event, base, third)
    rows = {row["uid"]: row for row in result["rows"]}
    assert rows["u0"]["prediction"] == "C"
    assert rows["u0"]["media_calls"] == 3
    assert rows["u0"]["model_processed_frames"] == 56
    assert rows["u0"]["unique_frames"] == 32
    assert rows["u0"]["visual_tokens"] == 60
    assert rows["u1"]["prediction"] == "D"
    assert rows["u1"]["third_view_policy"]["triggered"] is False
    assert result["policy"]["question_task_group_or_benchmark_router"] is False

	from experiments.materialize_third_view_disagreement_agent import materialize


	IDENTITY = {
	"benchmark": "demo",
	"model_manifest_sha256": "m",
	"annotation_sha256": "a",
	"evaluation_protocol_sha256": "p",
	"annotation": "annotation.json",
	"evaluation_protocol": {},
	"model": "model",
	"processor": "processor",
	}


	def _result(rows: list[dict]) -> dict:
	return {**IDENTITY, "rows": rows}


	def test_third_view_commits_only_on_acquired_disagreement() -> None:
	recent = _result(
	[
	{"uid": "u0", "prediction": "A", "visual_tokens": 10},
	{"uid": "u1", "prediction": "D", "visual_tokens": 10},
	]
	)
	event = _result(
	[
	{
	"uid": "u0",
	"prediction": "B",
	"error": None,
	"visual_tokens": 20,
	"sampling": {"frame_indices": list(range(20))},
	}
	]
	)
	base = _result(
	[
	{
	"uid": "u0",
	"prediction": "A",
	"agent_policy": {"escalated": True},
	"media_calls": 2,
	"num_turns": 2,
	"model_processed_frames": 24,
	"sampling": {"frame_indices": list(range(20))},
	},
	{"uid": "u1", "prediction": "D", "media_calls": 1},
	]
	)
	third = _result(
	[
	{
	"uid": "u0",
	"prediction": "C",
	"raw_response": "C",
	"error": None,
	"query_time": 31.0,
	"frames_observed": 32,
	"visual_tokens": 30,
	"sampling": {
	"frame_indices": list(range(32)),
	"frame_timestamps": list(map(float, range(32))),
	},
	}
	]
	)
	result = materialize(recent, event, base, third)
	rows = {row["uid"]: row for row in result["rows"]}
	assert rows["u0"]["prediction"] == "C"
	assert rows["u0"]["media_calls"] == 3
	assert rows["u0"]["model_processed_frames"] == 56
	assert rows["u0"]["unique_frames"] == 32
	assert rows["u0"]["visual_tokens"] == 60
	assert rows["u1"]["prediction"] == "D"
	assert rows["u1"]["third_view_policy"]["triggered"] is False
	assert result["policy"]["question_task_group_or_benchmark_router"] is False