"""HF processor for the PanoVLM decision model: a state and typed questions in, one model row per question out; option probabilities in, typed answers out. The interface is Laya Vision's ``predict``: * a state: a PIL image, a text, or a dict whose ``"image"`` / ``"images"`` keys hold images (PIL, path or encoded bytes) and whose other keys are the state's text, serialized to JSON (``{"context": ...}`` is the usual form); * questions: ``{name: {"type", "instructions", "criteria"}}`` with ``type`` ``choice`` (``criteria`` the options), ``score`` (``criteria`` the ordered level descriptions) or ``noul`` (yes / no, no ``criteria``). Each row is the PanoVLM prompt (chat template, images, state, typed question) followed by the options, one per line; the model reads each option's score at the newline closing its line. The rules are ``laya_core``'s, so every row follows the same rules. """ from __future__ import annotations import io import json import math from typing import Any import numpy as np from PIL import Image from transformers.feature_extraction_utils import BatchFeature # Two import layouts have to work, as for the modeling code: # * shipped: prepare_hf_upload copies the files below into the upload # directory beside this file, so the relative imports resolve on the Hub. # * in-repo: no copy exists here, so fall back to the package paths. try: from .laya_core import ( build_options_tail, fit_state_text, option_end_id, QUESTION_TYPE_IDS, render_options, render_question, render_state_text, ) from .processing_panovlm import PanoVLMProcessor except ImportError: # pragma: no cover - exercised only in-repo from torchtitan.models.panovlm.hf.processing_panovlm import PanoVLMProcessor from torchtitan.models.panovlm.system_1_decision.readouts.laya_core import ( build_options_tail, fit_state_text, option_end_id, QUESTION_TYPE_IDS, render_options, render_question, render_state_text, ) IMAGE_KEYS = ("image", "images") def _load_image(image) -> Image.Image: if isinstance(image, (bytes, bytearray)): image = Image.open(io.BytesIO(image)) elif isinstance(image, str): image = Image.open(image) return image.convert("RGB") def split_state(state: Any) -> tuple[list[Image.Image], str]: """``(images, state text)`` of a state (Laya's ``split_state``).""" if isinstance(state, Image.Image): return [state.convert("RGB")], "" if isinstance(state, str): return [], render_state_text(state) if not isinstance(state, dict): raise ValueError( f"a state is a PIL image, a text or a dict, got {type(state).__name__}" ) images = [] if state.get("image") is not None: images.append(_load_image(state["image"])) images += [_load_image(image) for image in state.get("images") or ()] rest = {k: v for k, v in state.items() if k not in IMAGE_KEYS} return images, json.dumps(rest, ensure_ascii=False) if rest else "" def option_labels(question: dict[str, Any]) -> list[str]: """How each option is named in the answer, in label order.""" if question["type"] == "noul": return ["false", "true"] if question["type"] == "score": return [str(i) for i in range(len(question["criteria"]))] return list(question["criteria"]) def _criteria(name: str, question: dict[str, Any]) -> list[str] | None: """The option texts the question type renders, validated.""" qtype = question.get("type") if qtype not in QUESTION_TYPE_IDS: raise ValueError( f"question {name!r}: type must be one of {list(QUESTION_TYPE_IDS)}, " f"got {qtype!r}" ) criteria = question.get("criteria") if qtype == "noul": if criteria is not None: raise ValueError( f"question {name!r}: a noul question takes no criteria " "(its options are always false / true)" ) return None if isinstance(criteria, dict) and qtype == "choice": # Laya's labelled options: "label: description". return [k if not v else f"{k}: {v}" for k, v in criteria.items()] if not criteria or len(criteria) < 2: raise ValueError(f"question {name!r}: {qtype} needs at least 2 criteria") return [str(c) for c in criteria] def _confidence(p: np.ndarray) -> float: """1 - normalized entropy (Laya's ``confidence_from_probs``).""" entropy = -(p * np.log(np.clip(p, 1e-12, 1.0))).sum() return float(np.clip(1.0 - entropy / math.log(len(p)), 0.0, 1.0)) class PanoVLMDecisionProcessor(PanoVLMProcessor): def __init__( self, image_processor=None, tokenizer=None, image_seq_length=None, chat_template=None, max_len: int = 1024, max_option_tokens: int = 48, **kwargs, ): # Laya's budgets: each option at most max_option_tokens; the state # text is shortened until the whole row fits max_len. self.max_len = max_len self.max_option_tokens = max_option_tokens super().__init__( image_processor, tokenizer, image_seq_length=image_seq_length, chat_template=chat_template, **kwargs, ) def _encode(self, text: str) -> list[int]: # The raw tokenizer, without BOS / EOS, as laya_core's rules expect. return self.tokenizer.backend_tokenizer.encode( text, add_special_tokens=False ).ids def _prompt_ids( self, num_images: int, question: dict[str, Any], state_text: str ) -> list[int]: instructions = question["instructions"] if not isinstance(instructions, str): instructions = json.dumps(instructions) text = render_question(question["type"], instructions, state_text) rendered = self.apply_chat_template( [{"role": "user", "content": self.image_token * num_images + text}], add_generation_prompt=True, tokenize=False, ) return self.tokenizer(self._prompt_text(rendered))["input_ids"] def _row( self, name: str, question: dict[str, Any], num_images: int, state_text: str ) -> tuple[list[int], list[int]]: """``(input_ids, markers)`` of one question, options in label order.""" options = render_options(question["type"], _criteria(name, question)) newline_id = option_end_id(self._encode) tail, markers, _ = build_options_tail( self._encode, options, range(len(options)), newline_id=newline_id, max_option_tokens=self.max_option_tokens, ) prompt = fit_state_text( lambda text: self._prompt_ids(num_images, question, text), len, state_text, tail_len=len(tail), max_len=self.max_len, name=f"question {name!r}", ) return prompt + tail, [m + len(prompt) for m in markers] def rows( self, state: Any, questions: dict[str, dict[str, Any]] ) -> tuple[list[Image.Image], list[tuple[list[int], list[int]]]]: """The state's images and ``(input_ids, markers)`` per question, in ``questions`` order: the rows ``__call__`` pads, before any pixel processing (the fast path resizes the images on the GPU).""" if state is None or not questions: raise ValueError("PanoVLMDecisionProcessor needs a state and questions.") images, state_text = split_state(state) rows = [ self._row(name, question, len(images), state_text) for name, question in questions.items() ] return images, rows def __call__( self, state: Any = None, questions: dict[str, dict[str, Any]] | None = None, return_tensors=None, **kwargs, ) -> BatchFeature: """One right-padded row per question, in ``questions`` order.""" images, rows = self.rows(state, questions) seq_len = max(len(ids) for ids, _ in rows) num_options = max(len(markers) for _, markers in rows) input_ids = np.full( (len(rows), seq_len), self.tokenizer.pad_token_id, dtype=np.int64 ) attention_mask = np.zeros((len(rows), seq_len), dtype=bool) marker_pos = np.zeros((len(rows), num_options), dtype=np.int64) marker_mask = np.zeros((len(rows), num_options), dtype=bool) for b, (ids, markers) in enumerate(rows): input_ids[b, : len(ids)] = ids attention_mask[b, : len(ids)] = True marker_pos[b, : len(markers)] = markers marker_mask[b, : len(markers)] = True data = { "input_ids": input_ids, "attention_mask": attention_mask, "marker_pos": marker_pos, "marker_mask": marker_mask, "question_type": np.array( [QUESTION_TYPE_IDS[q["type"]] for q in questions.values()] ), } if images: # Every row carries the state's images, in row order. pixel_values = self.image_processor(images)["pixel_values"] data["pixel_values"] = np.concatenate([pixel_values] * len(rows)) return BatchFeature(data=data, tensor_type=return_tensors) def post_process( self, probabilities, questions: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: """Typed answers in Laya Vision's schema, from the model's calibrated ``probabilities`` (one row per question, in ``questions`` order): * ``choice``: the most probable option, the probability of each, and a confidence (1 - normalized entropy); * ``score``: the expected level, the probability of each level, its legend and a confidence; * ``noul``: the probability of "true" (yes), and a confidence (the larger of the two probabilities). """ answers = {} for (name, question), probs in zip(questions.items(), probabilities): labels = option_labels(question) p = np.asarray(probs, dtype=np.float64)[: len(labels)] qtype = question["type"] if qtype == "noul": answers[name] = { "type": "noul", "noul": round(float(p[1]), 4), "confidence": round(max(float(p[1]), 1.0 - float(p[1])), 4), } continue answer = {"type": qtype} if qtype == "choice": answer["choice"] = labels[int(p.argmax())] else: answer["score"] = round(float((np.arange(len(p)) * p).sum()), 4) answer["legend"] = dict(zip(labels, question["criteria"])) answer["probabilities"] = { label: round(float(v), 4) for label, v in zip(labels, p) } answer["confidence"] = round(_confidence(p), 4) answers[name] = answer return answers