# -*- coding: utf-8 -*- """Multimodal tokenizer processor.""" from __future__ import annotations from dataclasses import dataclass, field from typing import Any from Tokenizer.unified.encoded import EncodedToken from .image_placeholders import ( image_patch_count, image_placeholder_tokens, ) from .video_placeholders import ( video_patch_count, video_placeholder_tokens, ) from .tokens import IMAGE_PLACEHOLDER, VIDEO_PLACEHOLDER @dataclass class MultimodalEncoding: input_ids: list[int] attention_mask: list[int] tokens: list[EncodedToken] image_token_spans: list[tuple[int, int]] video_token_spans: list[tuple[int, int]] = field(default_factory=list) images: list[Any] = field(default_factory=list) videos: list[Any] = field(default_factory=list) pixel_values: Any = None @dataclass(frozen=True) class _PlaceholderRecord: start: int end: int patch_count: int class MultimodalProcessor: def __init__( self, tokenizer, image_processor=None, patch_size: int = 14, merge_size: int = 2, temporal_patch_size: int = 2, ): self.tokenizer = tokenizer self.image_processor = image_processor self.patch_size = patch_size self.merge_size = merge_size self.temporal_patch_size = temporal_patch_size def __call__( self, text: str, images: list[Any] | None = None, image_sizes: list[Any] | None = None, videos: list[Any] | None = None, video_sizes: list[Any] | None = None, add_bos: bool = False, add_eos: bool = False, ) -> MultimodalEncoding: image_list = list(images or []) sizes = list(image_sizes or []) video_list = list(videos or []) vsizes = list(video_sizes or []) n_img_holes = text.count(IMAGE_PLACEHOLDER) n_vid_holes = text.count(VIDEO_PLACEHOLDER) if len(image_list) != n_img_holes: raise ValueError( f" placeholder count ({n_img_holes}) does not match " f"images length ({len(image_list)})" ) if len(video_list) != n_vid_holes: raise ValueError( f"