code-HFD / src /psi /config /transform_motion_distill.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
2.12 kB
from __future__ import annotations
from typing import Any
from psi.config.transform import Psi0ModelTransform
class MotionDistillPsi0ModelTransform(Psi0ModelTransform):
motion_latent_token: str = "<|motion_latent|>"
motion_latent_num_tokens: int = 64
def _append_motion_latent_tokens(
self,
instruction: Any,
*,
motion_latent_token: str | None = None,
motion_latent_num_tokens: int = 0,
) -> str:
instruction_text = str(instruction or "").strip()
token = str(motion_latent_token or self.motion_latent_token or "").strip()
n_tokens = int(motion_latent_num_tokens or self.motion_latent_num_tokens or 0)
if not token or n_tokens <= 0:
return instruction_text
latent_text = " ".join([token] * n_tokens)
if instruction_text:
return f"{instruction_text}\n{latent_text}"
return latent_text
def build_qwenvl_inputs(
self,
vlm_processor,
imgs,
instruction,
audio=None,
audio_path=None,
video=None,
video_path=None,
video_size_pixels=None,
video_sample_fps=1.0,
pixel_values_videos=None,
video_grid_thw=None,
video_second_per_grid=None,
motion_latent_token: str | None = None,
motion_latent_num_tokens: int = 0,
**kwargs,
):
instruction = self._append_motion_latent_tokens(
instruction,
motion_latent_token=motion_latent_token,
motion_latent_num_tokens=motion_latent_num_tokens,
)
return super().build_qwenvl_inputs(
vlm_processor,
imgs,
instruction,
audio=audio,
audio_path=audio_path,
video=video,
video_path=video_path,
video_size_pixels=video_size_pixels,
video_sample_fps=video_sample_fps,
pixel_values_videos=pixel_values_videos,
video_grid_thw=video_grid_thw,
video_second_per_grid=video_second_per_grid,
**kwargs,
)