File size: 8,050 Bytes
1f5791a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
import math
import warnings
from collections.abc import Iterable
import torch
from torchvision.transforms.v2 import functional as tvF
from transformers.image_processing_backends import TorchvisionBackend
from transformers.image_processing_utils import BatchFeature
from transformers.image_transforms import group_images_by_shape, reorder_images
from transformers.image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, PILImageResampling, SizeDict
from transformers.processing_utils import ImagesKwargs, Unpack
from transformers.utils import TensorType, auto_docstring

class AgnesImageProcessorKwargs(ImagesKwargs, total=False):
    min_pixels: int
    max_pixels: int
    patch_size: int
    temporal_patch_size: int
    merge_size: int

def smart_resize(height: int, width: int, factor: int=28, min_pixels: int=56 * 56, max_pixels: int=14 * 14 * 4 * 1280):
    if max(height, width) / min(height, width) > 200:
        raise ValueError(f'absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}')
    h_bar = round(height / factor) * factor
    w_bar = round(width / factor) * factor
    if h_bar * w_bar > max_pixels:
        beta = math.sqrt(height * width / max_pixels)
        h_bar = max(factor, math.floor(height / beta / factor) * factor)
        w_bar = max(factor, math.floor(width / beta / factor) * factor)
    elif h_bar * w_bar < min_pixels:
        beta = math.sqrt(min_pixels / (height * width))
        h_bar = math.ceil(height * beta / factor) * factor
        w_bar = math.ceil(width * beta / factor) * factor
    return (h_bar, w_bar)

class AgnesImageProcessor(TorchvisionBackend):
    do_resize = True
    resample = PILImageResampling.BICUBIC
    size = {'shortest_edge': 56 * 56, 'longest_edge': 28 * 28 * 1280}
    default_to_square = False
    do_rescale = True
    do_normalize = True
    image_mean = OPENAI_CLIP_MEAN
    image_std = OPENAI_CLIP_STD
    do_convert_rgb = True
    patch_size = 14
    temporal_patch_size = 2
    merge_size = 2
    valid_kwargs = AgnesImageProcessorKwargs
    model_input_names = ['pixel_values', 'image_grid_thw']

    def __init__(self, **kwargs: Unpack[AgnesImageProcessorKwargs]):
        size = kwargs.pop('size', None)
        size = dict(self.size) if size is None else size
        if (min_pixels := kwargs.pop('min_pixels', None)) is not None:
            size['shortest_edge'] = min_pixels
            size.pop('min_pixels', None)
        if (max_pixels := kwargs.pop('max_pixels', None)) is not None:
            size['longest_edge'] = max_pixels
            size.pop('max_pixels', None)
        super().__init__(size=size, **kwargs)

    def _standardize_kwargs(self, size: int | Iterable[int] | dict[str, int] | SizeDict | None=None, min_pixels: int | None=None, max_pixels: int | None=None, **kwargs) -> dict:
        if min_pixels is not None or max_pixels is not None:
            warnings.warn("Passing `min_pixels` and `max_pixels` to a processor call is deprecated and will be removed in v5.23. Pass in `size={'longest_edge': xxx, 'shortest_edge': xxx} to override the target size.`", FutureWarning)
            size_dict = dict(size) if isinstance(size, (dict, SizeDict)) else {}
            if min_pixels is not None:
                size_dict['shortest_edge'] = min_pixels
            if max_pixels is not None:
                size_dict['longest_edge'] = max_pixels
            size = SizeDict(**size_dict)
        return super()._standardize_kwargs(size=size, **kwargs)

    def preprocess(self, images: ImageInput, **kwargs: Unpack[AgnesImageProcessorKwargs]) -> BatchFeature:
        return super().preprocess(images, **kwargs)

    def resize(self, images: 'torch.Tensor', size: SizeDict, resample: 'PILImageResampling | tvF.InterpolationMode | int | None', factor: int, **kwargs) -> 'torch.Tensor':
        if not size.shortest_edge or not size.longest_edge:
            raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.")
        height, width = images.shape[-2:]
        resized_height, resized_width = smart_resize(height, width, factor=factor, min_pixels=size.shortest_edge, max_pixels=size.longest_edge)
        return super().resize(image=images, size=SizeDict(height=resized_height, width=resized_width), resample=resample)

    def patchify(self, images: 'torch.Tensor', patch_size: int, merge_size: int, temporal_patch_size: int) -> tuple['torch.Tensor', int, int]:
        batch_size, channel, resized_height, resized_width = images.shape
        grid_h, grid_w = (resized_height // patch_size, resized_width // patch_size)
        patches = images.reshape(batch_size, channel, grid_h // merge_size, merge_size, patch_size, grid_w // merge_size, merge_size, patch_size)
        patches = patches.permute(0, 2, 5, 3, 6, 1, 4, 7)
        flatten_patches = patches.unsqueeze(6).expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1).reshape(batch_size, grid_h * grid_w, channel * temporal_patch_size * patch_size * patch_size)
        return (flatten_patches, grid_h, grid_w)

    def _preprocess(self, images: list['torch.Tensor'], do_resize: bool, size: SizeDict, resample: 'PILImageResampling | tvF.InterpolationMode | int | None', do_rescale: bool, rescale_factor: float, do_normalize: bool, image_mean: float | list[float] | None, image_std: float | list[float] | None, patch_size: int, temporal_patch_size: int, merge_size: int, disable_grouping: bool | None, return_tensors: str | TensorType | None, **kwargs) -> BatchFeature:
        grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
        resized_images_grouped = {}
        for shape, stacked_images in grouped_images.items():
            if do_resize:
                stacked_images = self.resize(images=stacked_images, size=size, resample=resample, factor=patch_size * merge_size)
            resized_images_grouped[shape] = stacked_images
        resized_images = reorder_images(resized_images_grouped, grouped_images_index)
        grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
        processed_images_grouped = {}
        processed_grids = {}
        for shape, stacked_images in grouped_images.items():
            stacked_images = self.rescale_and_normalize(stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std)
            patches, grid_h, grid_w = self.patchify(stacked_images, patch_size=patch_size, merge_size=merge_size, temporal_patch_size=temporal_patch_size)
            processed_images_grouped[shape] = patches
            processed_grids[shape] = [[1, grid_h, grid_w]] * len(stacked_images)
        processed_images = reorder_images(processed_images_grouped, grouped_images_index)
        processed_grids_ordered = reorder_images(processed_grids, grouped_images_index)
        pixel_values = processed_images[0] if len(processed_images) == 1 else torch.cat(processed_images, dim=0)
        image_grid_thw = torch.tensor(processed_grids_ordered, dtype=torch.long)
        return BatchFeature(data={'pixel_values': pixel_values, 'image_grid_thw': image_grid_thw}, tensor_type=return_tensors)

    def get_number_of_image_patches(self, height: int, width: int, images_kwargs: dict | None=None) -> int:
        images_kwargs = images_kwargs or {}
        min_pixels = images_kwargs['min_pixels'] if 'min_pixels' in images_kwargs else self.size['shortest_edge']
        max_pixels = images_kwargs['max_pixels'] if 'max_pixels' in images_kwargs else self.size['longest_edge']
        patch_size = images_kwargs.get('patch_size', self.patch_size)
        merge_size = images_kwargs.get('merge_size', self.merge_size)
        factor = patch_size * merge_size
        resized_height, resized_width = smart_resize(height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels)
        grid_h, grid_w = (resized_height // patch_size, resized_width // patch_size)
        return grid_h * grid_w
__all__ = ['AgnesImageProcessor']