File size: 2,116 Bytes
c4f8e4a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
from __future__ import annotations

from typing import Any

from psi.config.transform import Psi0ModelTransform


class MotionDistillPsi0ModelTransform(Psi0ModelTransform):
    motion_latent_token: str = "<|motion_latent|>"
    motion_latent_num_tokens: int = 64

    def _append_motion_latent_tokens(
        self,
        instruction: Any,
        *,
        motion_latent_token: str | None = None,
        motion_latent_num_tokens: int = 0,
    ) -> str:
        instruction_text = str(instruction or "").strip()
        token = str(motion_latent_token or self.motion_latent_token or "").strip()
        n_tokens = int(motion_latent_num_tokens or self.motion_latent_num_tokens or 0)
        if not token or n_tokens <= 0:
            return instruction_text
        latent_text = " ".join([token] * n_tokens)
        if instruction_text:
            return f"{instruction_text}\n{latent_text}"
        return latent_text

    def build_qwenvl_inputs(
        self,
        vlm_processor,
        imgs,
        instruction,
        audio=None,
        audio_path=None,
        video=None,
        video_path=None,
        video_size_pixels=None,
        video_sample_fps=1.0,
        pixel_values_videos=None,
        video_grid_thw=None,
        video_second_per_grid=None,
        motion_latent_token: str | None = None,
        motion_latent_num_tokens: int = 0,
        **kwargs,
    ):
        instruction = self._append_motion_latent_tokens(
            instruction,
            motion_latent_token=motion_latent_token,
            motion_latent_num_tokens=motion_latent_num_tokens,
        )
        return super().build_qwenvl_inputs(
            vlm_processor,
            imgs,
            instruction,
            audio=audio,
            audio_path=audio_path,
            video=video,
            video_path=video_path,
            video_size_pixels=video_size_pixels,
            video_sample_fps=video_sample_fps,
            pixel_values_videos=pixel_values_videos,
            video_grid_thw=video_grid_thw,
            video_second_per_grid=video_second_per_grid,
            **kwargs,
        )