from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: from collections.abc import Callable from typing import Any from .api import PredictRequest from .backend import ARTask, EncodedRequest from transformers import PreTrainedTokenizerBase def eligible(request: PredictRequest) -> bool: if not request.state or not (request.images or request.documents): return False if len(request.images) > 1: return True from . import grounding return any( ( spec.get("type") == "boolean" and (not spec.get("knowledge")) and (not grounding.state_only(spec)) for spec in request.fields.values() ) ) def rewrite_media_evidence( task: ARTask, *, record: dict[str, Any], tokenizer: PreTrainedTokenizerBase, max_length: int, ) -> ARTask: from . import ar (name, ids, media) = task if media is None: raise ValueError("Media evidence guidance requires actual task media") source_ids = list(ids) text = tokenizer.decode( ids, skip_special_tokens=False, clean_up_tokenization_spaces=False ) if tokenizer.encode(text, add_special_tokens=False) != source_ids: raise ValueError( "Media evidence prompt must make a lossless tokenizer roundtrip" ) instruction = ar.instruction(record, name, compact_lists=False).rstrip() boundary = instruction + "\n\nDocument:\n" existing = ( ar.MEDIA_EVIDENCE_INSTRUCTION + "\n\n" + instruction + "\n\nAccompanying text:\n" ) if text.count(existing) == 1: return task if text.count(boundary) != 1 or ar.MEDIA_EVIDENCE_INSTRUCTION in text: raise ValueError( "Expected exactly one original instruction and accompanying-state boundary" ) revised = text.replace(boundary, existing) revised_ids = tokenizer.encode(revised, add_special_tokens=False) if ( tokenizer.decode( revised_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False ) != revised ): raise ValueError( "Media evidence revised prompt must make a lossless tokenizer roundtrip" ) if len(revised_ids) > max_length: raise ValueError("Media evidence prompt exceeds the complete model context") return (name, revised_ids, media) def prepare( request: PredictRequest, encoded: EncodedRequest, *, tokenizer: PreTrainedTokenizerBase, max_length: int, media_groups: Callable[..., list[dict[str, Any]]], ) -> EncodedRequest: if not eligible(request): return encoded from . import grounding (record, forecasts, tasks, typed, eos) = encoded annotations = ( record.get("_meta", {}).get("extraction_contract", {}).get("fields", {}) ) meta = record.get("_meta", {}) protected = set(meta.get("identifier_spelling", {})) | set( meta.get("model_po_supplement", {}).get("fields", ()) ) multi_image = len(request.images) > 1 updated: list[ARTask] = [] changed = False for task in tasks: (name, _, media) = task parent = annotations.get(name, {}).get("parent", name) spec = request.fields.get(parent) selected = ( media is not None and name not in protected and (parent not in protected) and (spec is not None) and (not spec.get("knowledge")) and (not grounding.state_only(spec)) and (multi_image or (name == parent and spec.get("type") == "boolean")) ) revised = task if selected: try: revised = rewrite_media_evidence( task, record=record, tokenizer=tokenizer, max_length=max_length ) except ValueError: return encoded updated.append(revised) changed |= revised is not task if not changed: return encoded revised_record = { **record, "_ar_tasks": updated, "_ar_prefix": [], "_ar_group_prefixes": [], } revised_record["_ar_media_groups"] = media_groups( updated, default_salt=record["_cache_salt"], task_salts=record.get("_ar_task_salts", {}), annotations=annotations, ) return (revised_record, forecasts, updated, typed, eos)