PanoVLD-500M / processing_panovlm_decision.py
sd24's picture
PanoVLD-500M
1ba3d87
Raw History Blame Contribute Delete
11.2 kB
"""HF processor for the PanoVLM decision model: a state and typed questions in,
one model row per question out; option probabilities in, typed answers out.
The interface is Laya Vision's ``predict``:
* a state: a PIL image, a text, or a dict whose ``"image"`` / ``"images"``
keys hold images (PIL, path or encoded bytes) and whose other keys are the
state's text, serialized to JSON (``{"context": ...}`` is the usual form);
* questions: ``{name: {"type", "instructions", "criteria"}}`` with ``type``
``choice`` (``criteria`` the options), ``score`` (``criteria`` the ordered
level descriptions) or ``noul`` (yes / no, no ``criteria``).
Each row is the PanoVLM prompt (chat template, images, state, typed question)
followed by the options, one per line; the model reads each option's score at
the newline closing its line. The rules are ``laya_core``'s, so every row
follows the same rules.
"""
from __future__ import annotations
import io
import json
import math
from typing import Any
import numpy as np
from PIL import Image
from transformers.feature_extraction_utils import BatchFeature
# Two import layouts have to work, as for the modeling code:
# * shipped: prepare_hf_upload copies the files below into the upload
# directory beside this file, so the relative imports resolve on the Hub.
# * in-repo: no copy exists here, so fall back to the package paths.
try:
from .laya_core import (
build_options_tail,
fit_state_text,
option_end_id,
QUESTION_TYPE_IDS,
render_options,
render_question,
render_state_text,
)
from .processing_panovlm import PanoVLMProcessor
except ImportError: # pragma: no cover - exercised only in-repo
from torchtitan.models.panovlm.hf.processing_panovlm import PanoVLMProcessor
from torchtitan.models.panovlm.system_1_decision.readouts.laya_core import (
build_options_tail,
fit_state_text,
option_end_id,
QUESTION_TYPE_IDS,
render_options,
render_question,
render_state_text,
)
IMAGE_KEYS = ("image", "images")
def _load_image(image) -> Image.Image:
if isinstance(image, (bytes, bytearray)):
image = Image.open(io.BytesIO(image))
elif isinstance(image, str):
image = Image.open(image)
return image.convert("RGB")
def split_state(state: Any) -> tuple[list[Image.Image], str]:
"""``(images, state text)`` of a state (Laya's ``split_state``)."""
if isinstance(state, Image.Image):
return [state.convert("RGB")], ""
if isinstance(state, str):
return [], render_state_text(state)
if not isinstance(state, dict):
raise ValueError(
f"a state is a PIL image, a text or a dict, got {type(state).__name__}"
)
images = []
if state.get("image") is not None:
images.append(_load_image(state["image"]))
images += [_load_image(image) for image in state.get("images") or ()]
rest = {k: v for k, v in state.items() if k not in IMAGE_KEYS}
return images, json.dumps(rest, ensure_ascii=False) if rest else ""
def option_labels(question: dict[str, Any]) -> list[str]:
"""How each option is named in the answer, in label order."""
if question["type"] == "noul":
return ["false", "true"]
if question["type"] == "score":
return [str(i) for i in range(len(question["criteria"]))]
return list(question["criteria"])
def _criteria(name: str, question: dict[str, Any]) -> list[str] | None:
"""The option texts the question type renders, validated."""
qtype = question.get("type")
if qtype not in QUESTION_TYPE_IDS:
raise ValueError(
f"question {name!r}: type must be one of {list(QUESTION_TYPE_IDS)}, "
f"got {qtype!r}"
)
criteria = question.get("criteria")
if qtype == "noul":
if criteria is not None:
raise ValueError(
f"question {name!r}: a noul question takes no criteria "
"(its options are always false / true)"
)
return None
if isinstance(criteria, dict) and qtype == "choice":
# Laya's labelled options: "label: description".
return [k if not v else f"{k}: {v}" for k, v in criteria.items()]
if not criteria or len(criteria) < 2:
raise ValueError(f"question {name!r}: {qtype} needs at least 2 criteria")
return [str(c) for c in criteria]
def _confidence(p: np.ndarray) -> float:
"""1 - normalized entropy (Laya's ``confidence_from_probs``)."""
entropy = -(p * np.log(np.clip(p, 1e-12, 1.0))).sum()
return float(np.clip(1.0 - entropy / math.log(len(p)), 0.0, 1.0))
class PanoVLMDecisionProcessor(PanoVLMProcessor):
def __init__(
self,
image_processor=None,
tokenizer=None,
image_seq_length=None,
chat_template=None,
max_len: int = 1024,
max_option_tokens: int = 48,
**kwargs,
):
# Laya's budgets: each option at most max_option_tokens; the state
# text is shortened until the whole row fits max_len.
self.max_len = max_len
self.max_option_tokens = max_option_tokens
super().__init__(
image_processor,
tokenizer,
image_seq_length=image_seq_length,
chat_template=chat_template,
**kwargs,
)
def _encode(self, text: str) -> list[int]:
# The raw tokenizer, without BOS / EOS, as laya_core's rules expect.
return self.tokenizer.backend_tokenizer.encode(
text, add_special_tokens=False
).ids
def _prompt_ids(
self, num_images: int, question: dict[str, Any], state_text: str
) -> list[int]:
instructions = question["instructions"]
if not isinstance(instructions, str):
instructions = json.dumps(instructions)
text = render_question(question["type"], instructions, state_text)
rendered = self.apply_chat_template(
[{"role": "user", "content": self.image_token * num_images + text}],
add_generation_prompt=True,
tokenize=False,
)
return self.tokenizer(self._prompt_text(rendered))["input_ids"]
def _row(
self, name: str, question: dict[str, Any], num_images: int, state_text: str
) -> tuple[list[int], list[int]]:
"""``(input_ids, markers)`` of one question, options in label order."""
options = render_options(question["type"], _criteria(name, question))
newline_id = option_end_id(self._encode)
tail, markers, _ = build_options_tail(
self._encode,
options,
range(len(options)),
newline_id=newline_id,
max_option_tokens=self.max_option_tokens,
)
prompt = fit_state_text(
lambda text: self._prompt_ids(num_images, question, text),
len,
state_text,
tail_len=len(tail),
max_len=self.max_len,
name=f"question {name!r}",
)
return prompt + tail, [m + len(prompt) for m in markers]
def rows(
self, state: Any, questions: dict[str, dict[str, Any]]
) -> tuple[list[Image.Image], list[tuple[list[int], list[int]]]]:
"""The state's images and ``(input_ids, markers)`` per question, in
``questions`` order: the rows ``__call__`` pads, before any pixel
processing (the fast path resizes the images on the GPU)."""
if state is None or not questions:
raise ValueError("PanoVLMDecisionProcessor needs a state and questions.")
images, state_text = split_state(state)
rows = [
self._row(name, question, len(images), state_text)
for name, question in questions.items()
]
return images, rows
def __call__(
self,
state: Any = None,
questions: dict[str, dict[str, Any]] | None = None,
return_tensors=None,
**kwargs,
) -> BatchFeature:
"""One right-padded row per question, in ``questions`` order."""
images, rows = self.rows(state, questions)
seq_len = max(len(ids) for ids, _ in rows)
num_options = max(len(markers) for _, markers in rows)
input_ids = np.full(
(len(rows), seq_len), self.tokenizer.pad_token_id, dtype=np.int64
)
attention_mask = np.zeros((len(rows), seq_len), dtype=bool)
marker_pos = np.zeros((len(rows), num_options), dtype=np.int64)
marker_mask = np.zeros((len(rows), num_options), dtype=bool)
for b, (ids, markers) in enumerate(rows):
input_ids[b, : len(ids)] = ids
attention_mask[b, : len(ids)] = True
marker_pos[b, : len(markers)] = markers
marker_mask[b, : len(markers)] = True
data = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"marker_pos": marker_pos,
"marker_mask": marker_mask,
"question_type": np.array(
[QUESTION_TYPE_IDS[q["type"]] for q in questions.values()]
),
}
if images:
# Every row carries the state's images, in row order.
pixel_values = self.image_processor(images)["pixel_values"]
data["pixel_values"] = np.concatenate([pixel_values] * len(rows))
return BatchFeature(data=data, tensor_type=return_tensors)
def post_process(
self, probabilities, questions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
"""Typed answers in Laya Vision's schema, from the model's calibrated
``probabilities`` (one row per question, in ``questions`` order):
* ``choice``: the most probable option, the probability of each, and a
confidence (1 - normalized entropy);
* ``score``: the expected level, the probability of each level, its
legend and a confidence;
* ``noul``: the probability of "true" (yes), and a confidence (the
larger of the two probabilities).
"""
answers = {}
for (name, question), probs in zip(questions.items(), probabilities):
labels = option_labels(question)
p = np.asarray(probs, dtype=np.float64)[: len(labels)]
qtype = question["type"]
if qtype == "noul":
answers[name] = {
"type": "noul",
"noul": round(float(p[1]), 4),
"confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4),
}
continue
answer = {"type": qtype}
if qtype == "choice":
answer["choice"] = labels[int(p.argmax())]
else:
answer["score"] = round(float((np.arange(len(p)) * p).sum()), 4)
answer["legend"] = dict(zip(labels, question["criteria"]))
answer["probabilities"] = {
label: round(float(v), 4) for label, v in zip(labels, p)
}
answer["confidence"] = round(_confidence(p), 4)
answers[name] = answer
return answers