Visual Question Answering
Transformers
Safetensors
English
panovlm_decision
feature-extraction
panovlm
decision-model
calibration
fastvit
linear-attention
custom_code
Instructions to use PanocularAI/PanoVLD-500M with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use PanocularAI/PanoVLD-500M with Transformers:
# Use a pipeline as a high-level helper # Warning: Pipeline type "visual-question-answering" is no longer supported in transformers v5. # You must load the model directly (see below) or downgrade to v4.x with: # pip install "transformers<5.0.0" from transformers import pipeline pipe = pipeline("visual-question-answering", model="PanocularAI/PanoVLD-500M", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("PanocularAI/PanoVLD-500M", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download processing_panovlm_decision.py from PanocularAI/PanoVLD-500M: direct link, hf CLI and curl.
- Browser
- Download file 11.2 kB
-
https://huggingface.co/PanocularAI/PanoVLD-500M/resolve/main/processing_panovlm_decision.py
- Command line
-
hf download hf://PanocularAI/PanoVLD-500M/processing_panovlm_decision.py
-
curl -L -o processing_panovlm_decision.py https://huggingface.co/PanocularAI/PanoVLD-500M/resolve/main/processing_panovlm_decision.py
11.2 kB
| """HF processor for the PanoVLM decision model: a state and typed questions in, | |
| one model row per question out; option probabilities in, typed answers out. | |
| The interface is Laya Vision's ``predict``: | |
| * a state: a PIL image, a text, or a dict whose ``"image"`` / ``"images"`` | |
| keys hold images (PIL, path or encoded bytes) and whose other keys are the | |
| state's text, serialized to JSON (``{"context": ...}`` is the usual form); | |
| * questions: ``{name: {"type", "instructions", "criteria"}}`` with ``type`` | |
| ``choice`` (``criteria`` the options), ``score`` (``criteria`` the ordered | |
| level descriptions) or ``noul`` (yes / no, no ``criteria``). | |
| Each row is the PanoVLM prompt (chat template, images, state, typed question) | |
| followed by the options, one per line; the model reads each option's score at | |
| the newline closing its line. The rules are ``laya_core``'s, so every row | |
| follows the same rules. | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import json | |
| import math | |
| from typing import Any | |
| import numpy as np | |
| from PIL import Image | |
| from transformers.feature_extraction_utils import BatchFeature | |
| # Two import layouts have to work, as for the modeling code: | |
| # * shipped: prepare_hf_upload copies the files below into the upload | |
| # directory beside this file, so the relative imports resolve on the Hub. | |
| # * in-repo: no copy exists here, so fall back to the package paths. | |
| try: | |
| from .laya_core import ( | |
| build_options_tail, | |
| fit_state_text, | |
| option_end_id, | |
| QUESTION_TYPE_IDS, | |
| render_options, | |
| render_question, | |
| render_state_text, | |
| ) | |
| from .processing_panovlm import PanoVLMProcessor | |
| except ImportError: # pragma: no cover - exercised only in-repo | |
| from torchtitan.models.panovlm.hf.processing_panovlm import PanoVLMProcessor | |
| from torchtitan.models.panovlm.system_1_decision.readouts.laya_core import ( | |
| build_options_tail, | |
| fit_state_text, | |
| option_end_id, | |
| QUESTION_TYPE_IDS, | |
| render_options, | |
| render_question, | |
| render_state_text, | |
| ) | |
| IMAGE_KEYS = ("image", "images") | |
| def _load_image(image) -> Image.Image: | |
| if isinstance(image, (bytes, bytearray)): | |
| image = Image.open(io.BytesIO(image)) | |
| elif isinstance(image, str): | |
| image = Image.open(image) | |
| return image.convert("RGB") | |
| def split_state(state: Any) -> tuple[list[Image.Image], str]: | |
| """``(images, state text)`` of a state (Laya's ``split_state``).""" | |
| if isinstance(state, Image.Image): | |
| return [state.convert("RGB")], "" | |
| if isinstance(state, str): | |
| return [], render_state_text(state) | |
| if not isinstance(state, dict): | |
| raise ValueError( | |
| f"a state is a PIL image, a text or a dict, got {type(state).__name__}" | |
| ) | |
| images = [] | |
| if state.get("image") is not None: | |
| images.append(_load_image(state["image"])) | |
| images += [_load_image(image) for image in state.get("images") or ()] | |
| rest = {k: v for k, v in state.items() if k not in IMAGE_KEYS} | |
| return images, json.dumps(rest, ensure_ascii=False) if rest else "" | |
| def option_labels(question: dict[str, Any]) -> list[str]: | |
| """How each option is named in the answer, in label order.""" | |
| if question["type"] == "noul": | |
| return ["false", "true"] | |
| if question["type"] == "score": | |
| return [str(i) for i in range(len(question["criteria"]))] | |
| return list(question["criteria"]) | |
| def _criteria(name: str, question: dict[str, Any]) -> list[str] | None: | |
| """The option texts the question type renders, validated.""" | |
| qtype = question.get("type") | |
| if qtype not in QUESTION_TYPE_IDS: | |
| raise ValueError( | |
| f"question {name!r}: type must be one of {list(QUESTION_TYPE_IDS)}, " | |
| f"got {qtype!r}" | |
| ) | |
| criteria = question.get("criteria") | |
| if qtype == "noul": | |
| if criteria is not None: | |
| raise ValueError( | |
| f"question {name!r}: a noul question takes no criteria " | |
| "(its options are always false / true)" | |
| ) | |
| return None | |
| if isinstance(criteria, dict) and qtype == "choice": | |
| # Laya's labelled options: "label: description". | |
| return [k if not v else f"{k}: {v}" for k, v in criteria.items()] | |
| if not criteria or len(criteria) < 2: | |
| raise ValueError(f"question {name!r}: {qtype} needs at least 2 criteria") | |
| return [str(c) for c in criteria] | |
| def _confidence(p: np.ndarray) -> float: | |
| """1 - normalized entropy (Laya's ``confidence_from_probs``).""" | |
| entropy = -(p * np.log(np.clip(p, 1e-12, 1.0))).sum() | |
| return float(np.clip(1.0 - entropy / math.log(len(p)), 0.0, 1.0)) | |
| class PanoVLMDecisionProcessor(PanoVLMProcessor): | |
| def __init__( | |
| self, | |
| image_processor=None, | |
| tokenizer=None, | |
| image_seq_length=None, | |
| chat_template=None, | |
| max_len: int = 1024, | |
| max_option_tokens: int = 48, | |
| **kwargs, | |
| ): | |
| # Laya's budgets: each option at most max_option_tokens; the state | |
| # text is shortened until the whole row fits max_len. | |
| self.max_len = max_len | |
| self.max_option_tokens = max_option_tokens | |
| super().__init__( | |
| image_processor, | |
| tokenizer, | |
| image_seq_length=image_seq_length, | |
| chat_template=chat_template, | |
| **kwargs, | |
| ) | |
| def _encode(self, text: str) -> list[int]: | |
| # The raw tokenizer, without BOS / EOS, as laya_core's rules expect. | |
| return self.tokenizer.backend_tokenizer.encode( | |
| text, add_special_tokens=False | |
| ).ids | |
| def _prompt_ids( | |
| self, num_images: int, question: dict[str, Any], state_text: str | |
| ) -> list[int]: | |
| instructions = question["instructions"] | |
| if not isinstance(instructions, str): | |
| instructions = json.dumps(instructions) | |
| text = render_question(question["type"], instructions, state_text) | |
| rendered = self.apply_chat_template( | |
| [{"role": "user", "content": self.image_token * num_images + text}], | |
| add_generation_prompt=True, | |
| tokenize=False, | |
| ) | |
| return self.tokenizer(self._prompt_text(rendered))["input_ids"] | |
| def _row( | |
| self, name: str, question: dict[str, Any], num_images: int, state_text: str | |
| ) -> tuple[list[int], list[int]]: | |
| """``(input_ids, markers)`` of one question, options in label order.""" | |
| options = render_options(question["type"], _criteria(name, question)) | |
| newline_id = option_end_id(self._encode) | |
| tail, markers, _ = build_options_tail( | |
| self._encode, | |
| options, | |
| range(len(options)), | |
| newline_id=newline_id, | |
| max_option_tokens=self.max_option_tokens, | |
| ) | |
| prompt = fit_state_text( | |
| lambda text: self._prompt_ids(num_images, question, text), | |
| len, | |
| state_text, | |
| tail_len=len(tail), | |
| max_len=self.max_len, | |
| name=f"question {name!r}", | |
| ) | |
| return prompt + tail, [m + len(prompt) for m in markers] | |
| def rows( | |
| self, state: Any, questions: dict[str, dict[str, Any]] | |
| ) -> tuple[list[Image.Image], list[tuple[list[int], list[int]]]]: | |
| """The state's images and ``(input_ids, markers)`` per question, in | |
| ``questions`` order: the rows ``__call__`` pads, before any pixel | |
| processing (the fast path resizes the images on the GPU).""" | |
| if state is None or not questions: | |
| raise ValueError("PanoVLMDecisionProcessor needs a state and questions.") | |
| images, state_text = split_state(state) | |
| rows = [ | |
| self._row(name, question, len(images), state_text) | |
| for name, question in questions.items() | |
| ] | |
| return images, rows | |
| def __call__( | |
| self, | |
| state: Any = None, | |
| questions: dict[str, dict[str, Any]] | None = None, | |
| return_tensors=None, | |
| **kwargs, | |
| ) -> BatchFeature: | |
| """One right-padded row per question, in ``questions`` order.""" | |
| images, rows = self.rows(state, questions) | |
| seq_len = max(len(ids) for ids, _ in rows) | |
| num_options = max(len(markers) for _, markers in rows) | |
| input_ids = np.full( | |
| (len(rows), seq_len), self.tokenizer.pad_token_id, dtype=np.int64 | |
| ) | |
| attention_mask = np.zeros((len(rows), seq_len), dtype=bool) | |
| marker_pos = np.zeros((len(rows), num_options), dtype=np.int64) | |
| marker_mask = np.zeros((len(rows), num_options), dtype=bool) | |
| for b, (ids, markers) in enumerate(rows): | |
| input_ids[b, : len(ids)] = ids | |
| attention_mask[b, : len(ids)] = True | |
| marker_pos[b, : len(markers)] = markers | |
| marker_mask[b, : len(markers)] = True | |
| data = { | |
| "input_ids": input_ids, | |
| "attention_mask": attention_mask, | |
| "marker_pos": marker_pos, | |
| "marker_mask": marker_mask, | |
| "question_type": np.array( | |
| [QUESTION_TYPE_IDS[q["type"]] for q in questions.values()] | |
| ), | |
| } | |
| if images: | |
| # Every row carries the state's images, in row order. | |
| pixel_values = self.image_processor(images)["pixel_values"] | |
| data["pixel_values"] = np.concatenate([pixel_values] * len(rows)) | |
| return BatchFeature(data=data, tensor_type=return_tensors) | |
| def post_process( | |
| self, probabilities, questions: dict[str, dict[str, Any]] | |
| ) -> dict[str, dict[str, Any]]: | |
| """Typed answers in Laya Vision's schema, from the model's calibrated | |
| ``probabilities`` (one row per question, in ``questions`` order): | |
| * ``choice``: the most probable option, the probability of each, and a | |
| confidence (1 - normalized entropy); | |
| * ``score``: the expected level, the probability of each level, its | |
| legend and a confidence; | |
| * ``noul``: the probability of "true" (yes), and a confidence (the | |
| larger of the two probabilities). | |
| """ | |
| answers = {} | |
| for (name, question), probs in zip(questions.items(), probabilities): | |
| labels = option_labels(question) | |
| p = np.asarray(probs, dtype=np.float64)[: len(labels)] | |
| qtype = question["type"] | |
| if qtype == "noul": | |
| answers[name] = { | |
| "type": "noul", | |
| "noul": round(float(p[1]), 4), | |
| "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4), | |
| } | |
| continue | |
| answer = {"type": qtype} | |
| if qtype == "choice": | |
| answer["choice"] = labels[int(p.argmax())] | |
| else: | |
| answer["score"] = round(float((np.arange(len(p)) * p).sum()), 4) | |
| answer["legend"] = dict(zip(labels, question["criteria"])) | |
| answer["probabilities"] = { | |
| label: round(float(v), 4) for label, v in zip(labels, p) | |
| } | |
| answer["confidence"] = round(_confidence(p), 4) | |
| answers[name] = answer | |
| return answers | |