File size: 2,116 Bytes
c4f8e4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | from __future__ import annotations
from typing import Any
from psi.config.transform import Psi0ModelTransform
class MotionDistillPsi0ModelTransform(Psi0ModelTransform):
motion_latent_token: str = "<|motion_latent|>"
motion_latent_num_tokens: int = 64
def _append_motion_latent_tokens(
self,
instruction: Any,
*,
motion_latent_token: str | None = None,
motion_latent_num_tokens: int = 0,
) -> str:
instruction_text = str(instruction or "").strip()
token = str(motion_latent_token or self.motion_latent_token or "").strip()
n_tokens = int(motion_latent_num_tokens or self.motion_latent_num_tokens or 0)
if not token or n_tokens <= 0:
return instruction_text
latent_text = " ".join([token] * n_tokens)
if instruction_text:
return f"{instruction_text}\n{latent_text}"
return latent_text
def build_qwenvl_inputs(
self,
vlm_processor,
imgs,
instruction,
audio=None,
audio_path=None,
video=None,
video_path=None,
video_size_pixels=None,
video_sample_fps=1.0,
pixel_values_videos=None,
video_grid_thw=None,
video_second_per_grid=None,
motion_latent_token: str | None = None,
motion_latent_num_tokens: int = 0,
**kwargs,
):
instruction = self._append_motion_latent_tokens(
instruction,
motion_latent_token=motion_latent_token,
motion_latent_num_tokens=motion_latent_num_tokens,
)
return super().build_qwenvl_inputs(
vlm_processor,
imgs,
instruction,
audio=audio,
audio_path=audio_path,
video=video,
video_path=video_path,
video_size_pixels=video_size_pixels,
video_sample_fps=video_sample_fps,
pixel_values_videos=pixel_values_videos,
video_grid_thw=video_grid_thw,
video_second_per_grid=video_second_per_grid,
**kwargs,
)
|