from __future__ import annotations from typing import Any from psi.config.transform import Psi0ModelTransform class MotionDistillPsi0ModelTransform(Psi0ModelTransform): motion_latent_token: str = "<|motion_latent|>" motion_latent_num_tokens: int = 64 def _append_motion_latent_tokens( self, instruction: Any, *, motion_latent_token: str | None = None, motion_latent_num_tokens: int = 0, ) -> str: instruction_text = str(instruction or "").strip() token = str(motion_latent_token or self.motion_latent_token or "").strip() n_tokens = int(motion_latent_num_tokens or self.motion_latent_num_tokens or 0) if not token or n_tokens <= 0: return instruction_text latent_text = " ".join([token] * n_tokens) if instruction_text: return f"{instruction_text}\n{latent_text}" return latent_text def build_qwenvl_inputs( self, vlm_processor, imgs, instruction, audio=None, audio_path=None, video=None, video_path=None, video_size_pixels=None, video_sample_fps=1.0, pixel_values_videos=None, video_grid_thw=None, video_second_per_grid=None, motion_latent_token: str | None = None, motion_latent_num_tokens: int = 0, **kwargs, ): instruction = self._append_motion_latent_tokens( instruction, motion_latent_token=motion_latent_token, motion_latent_num_tokens=motion_latent_num_tokens, ) return super().build_qwenvl_inputs( vlm_processor, imgs, instruction, audio=audio, audio_path=audio_path, video=video, video_path=video_path, video_size_pixels=video_size_pixels, video_sample_fps=video_sample_fps, pixel_values_videos=pixel_values_videos, video_grid_thw=video_grid_thw, video_second_per_grid=video_second_per_grid, **kwargs, )