File size: 3,807 Bytes
1889fc4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
"""Processor glue for GroundingPI with Kimi-K3 MoonViT3D preprocessing."""

from transformers.feature_extraction_utils import BatchFeature

from .media_utils import MediaInput
from .image_processing_groundingpi import GroundingPIImageProcessor


class GroundingPIProcessor:
    attributes = ["image_processor", "tokenizer"]
    image_processor_class = "AutoImageProcessor"
    tokenizer_class = "AutoTokenizer"

    def __init__(self, image_processor=None, tokenizer=None, chat_template=None, **kwargs):
        del kwargs
        self.image_processor = image_processor
        self.video_processor = image_processor
        self.tokenizer = tokenizer
        self.chat_template = chat_template or getattr(tokenizer, "chat_template", None)

    @classmethod
    def register_for_auto_class(cls, auto_class="AutoProcessor"):
        cls._auto_class = auto_class

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
        import json
        import os
        from transformers import AutoTokenizer

        kwargs.pop("_from_auto", None)
        kwargs.pop("trust_remote_code", None)
        kwargs.pop("code_revision", None)
        with open(os.path.join(pretrained_model_name_or_path, "preprocessor_config.json"), encoding="utf-8") as f:
            processor_config = json.load(f)
        image_processor = GroundingPIImageProcessor(
            media_proc_cfg=processor_config["media_proc_cfg"]
        )
        tokenizer = AutoTokenizer.from_pretrained(
            pretrained_model_name_or_path, trust_remote_code=True, **kwargs
        )
        return cls(image_processor=image_processor, tokenizer=tokenizer)

    def apply_chat_template(self, messages, **kwargs):
        if self.chat_template and "chat_template" not in kwargs:
            kwargs["chat_template"] = self.chat_template
        return self.tokenizer.apply_chat_template(messages, **kwargs)

    def __call__(
        self,
        text=None,
        images=None,
        videos=None,
        return_tensors="pt",
        padding=False,
        video_mode="inference",
        video_fps=None,
        video_high_motion=None,
        video_auto_high_motion=None,
        video_max_tokens=None,
        video_max_tokens_per_frame=None,
        video_max_frames=None,
        video_sampling_seed=None,
        return_video_metadata=False,
        **kwargs,
    ):
        if isinstance(text, str):
            text = [text]
        text_inputs = self.tokenizer(
            text,
            return_tensors=return_tensors,
            padding=padding,
            **kwargs,
        )
        data = dict(text_inputs)
        if images is not None:
            image_inputs = self.image_processor(
                images=images,
                return_tensors=return_tensors,
            )
            data.update(image_inputs)
        if videos is not None:
            video_inputs = self.video_processor(
                videos=videos,
                return_tensors=return_tensors,
                video_mode=video_mode,
                video_fps=video_fps,
                video_high_motion=video_high_motion,
                video_auto_high_motion=video_auto_high_motion,
                video_max_tokens=video_max_tokens,
                video_max_tokens_per_frame=video_max_tokens_per_frame,
                video_max_frames=video_max_frames,
                video_sampling_seed=video_sampling_seed,
                return_video_metadata=return_video_metadata,
            )
            data.update(video_inputs)
        return BatchFeature(data=data)

    def batch_decode(self, *args, **kwargs):
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        return self.tokenizer.decode(*args, **kwargs)


__all__ = ["GroundingPIProcessor"]