Spaces:
Running on Zero
Running on Zero
File size: 18,782 Bytes
3db1cbc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 | # Copyright 2026 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
from ..modular_pipeline import AutoPipelineBlocks, SequentialPipelineBlocks
from ..modular_pipeline_utils import OutputParam
from .decoders import (
LTX2AudioDecoderStep,
LTX2DiffusionVaeDecoderStep,
LTX2TrimConditionTokensStep,
)
from .modular_blocks_ltx2 import (
LTX2AutoConditionEncoderStep,
LTX2AutoCoreDenoiseStep,
LTX2AutoDurationStep,
LTX2AutoPromptEnhancerStep,
LTX2AutoReferenceEncoderStep,
LTX2AutoVaeEncoderStep,
LTX2TextConditioningStep,
)
# auto_docstring
class LTX25DecoderStep(SequentialPipelineBlocks):
"""
Decode stage for LTX-2.5: denoises the video latents with the diffusion decoder and vocodes the audio latents (or
returns latents).
Components:
diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae
(`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`)
Inputs:
latents (`Tensor`):
Pre-generated noisy latents for image generation.
output_type (`str`, *optional*, defaults to pil):
Output format: 'pil', 'np', 'pt'.
height (`int`, *optional*, defaults to 512):
The height in pixels of the generated image.
width (`int`, *optional*, defaults to 704):
The width in pixels of the generated image.
num_frames (`int`, *optional*, defaults to 121):
The number of frames in the generated video.
generator (`Generator`, *optional*):
Torch generator for deterministic generation.
dtype (`dtype`):
The dtype of the model inputs, can be generated in input step.
audio_latents (`Tensor`):
Denoised audio latents.
audio_num_frames (`int`):
Number of audio latent frames, used to unpack the audio latent sequence.
Outputs:
videos (`list`):
The generated videos.
audio (`Tensor`):
The generated audio waveform.
"""
model_name = "ltx2.5"
block_classes = [LTX2DiffusionVaeDecoderStep, LTX2AudioDecoderStep]
block_names = ["video_decode", "audio_decode"]
@property
def description(self):
return (
"Decode stage for LTX-2.5: denoises the video latents with the diffusion decoder and vocodes the audio "
"latents (or returns latents)."
)
@property
def outputs(self):
return [
OutputParam.template("videos"),
OutputParam("audio", type_hint=torch.Tensor, description="The generated audio waveform."),
]
# auto_docstring
class LTX25ConditionDecoderStep(SequentialPipelineBlocks):
"""
Decode stage for LTX-2.5 condition workflows: drops the appended keyframe-condition tokens, then denoises the video
latents with the diffusion decoder and vocodes the audio latents (or returns latents).
Components:
diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae
(`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`)
Inputs:
latents (`Tensor`):
Pre-generated noisy latents for image generation.
base_token_count (`int`):
Number of generated-video tokens, i.e. the sequence length before appended tokens.
output_type (`str`, *optional*, defaults to pil):
Output format: 'pil', 'np', 'pt'.
height (`int`, *optional*, defaults to 512):
The height in pixels of the generated image.
width (`int`, *optional*, defaults to 704):
The width in pixels of the generated image.
num_frames (`int`, *optional*, defaults to 121):
The number of frames in the generated video.
generator (`Generator`, *optional*):
Torch generator for deterministic generation.
dtype (`dtype`):
The dtype of the model inputs, can be generated in input step.
audio_latents (`Tensor`):
Denoised audio latents.
audio_num_frames (`int`):
Number of audio latent frames, used to unpack the audio latent sequence.
Outputs:
videos (`list`):
The generated videos.
audio (`Tensor`):
The generated audio waveform.
"""
model_name = "ltx2.5"
block_classes = [LTX2TrimConditionTokensStep, LTX2DiffusionVaeDecoderStep, LTX2AudioDecoderStep]
block_names = ["trim_condition_tokens", "video_decode", "audio_decode"]
@property
def description(self):
return (
"Decode stage for LTX-2.5 condition workflows: drops the appended keyframe-condition tokens, then "
"denoises the video latents with the diffusion decoder and vocodes the audio latents (or returns "
"latents)."
)
@property
def outputs(self):
return [
OutputParam.template("videos"),
OutputParam("audio", type_hint=torch.Tensor, description="The generated audio waveform."),
]
# auto_docstring
class LTX25AutoDecoderStep(AutoPipelineBlocks):
"""
Auto decode block for LTX-2.5 that selects the decoder based on inputs.
- `LTX25ConditionDecoderStep` when `base_token_count` is present, i.e. the denoised sequence carries appended
keyframe / reference tokens (condition, in-context).
- `LTX25DecoderStep` otherwise (text-to-video, image-to-video).
Components:
diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) video_processor (`VideoProcessor`) audio_vae
(`AutoencoderKLLTX2Audio`) vocoder (`LTX2Vocoder`)
Inputs:
latents (`Tensor`):
Pre-generated noisy latents for image generation.
base_token_count (`int`, *optional*):
Number of generated-video tokens, i.e. the sequence length before appended tokens.
output_type (`str`, *optional*, defaults to pil):
Output format: 'pil', 'np', 'pt'.
height (`int`, *optional*, defaults to 512):
The height in pixels of the generated image.
width (`int`, *optional*, defaults to 704):
The width in pixels of the generated image.
num_frames (`int`, *optional*, defaults to 121):
The number of frames in the generated video.
generator (`Generator`, *optional*):
Torch generator for deterministic generation.
dtype (`dtype`):
The dtype of the model inputs, can be generated in input step.
audio_latents (`Tensor`):
Denoised audio latents.
audio_num_frames (`int`):
Number of audio latent frames, used to unpack the audio latent sequence.
Outputs:
videos (`list`):
The generated videos.
audio (`Tensor`):
The generated audio waveform.
"""
model_name = "ltx2.5"
# Mirrors `LTX2AutoDecoderStep` on the same `base_token_count` trigger; only the video decoder differs.
block_classes = [LTX25ConditionDecoderStep, LTX25DecoderStep]
block_names = ["condition", "default"]
block_trigger_inputs = ["base_token_count", None]
@property
def description(self):
return (
"Auto decode block for LTX-2.5 that selects the decoder based on inputs.\n"
" - `LTX25ConditionDecoderStep` when `base_token_count` is present, i.e. the denoised sequence carries "
"appended keyframe / reference tokens (condition, in-context).\n"
" - `LTX25DecoderStep` otherwise (text-to-video, image-to-video)."
)
@property
def outputs(self):
return [
OutputParam.template("videos"),
OutputParam("audio", type_hint=torch.Tensor, description="The generated audio waveform."),
]
# auto_docstring
class LTX25AutoBlocks(SequentialPipelineBlocks):
"""
Auto blocks for LTX-2.5 supporting text-to-video, image-to-video, condition-to-video and in-context (IC-LoRA)
generation (joint video + audio). Identical to `LTX2AutoBlocks` except that the video decoder is
`LTX2DiffusionVaeDecoderStep`, since the diffusion decoder is the native default from LTX-2.5 on. To decode with
the convolutional VAE instead, swap the decode block: `blocks.sub_blocks["decode"] = LTX2AutoDecoderStep()`.
Supported workflows:
- `text2video`: requires `prompt`
- `image2video`: requires `image`, `prompt`
- `condition`: requires `conditions`, `prompt`
- `in_context`: requires `reference_conditions`, `num_frames`, `prompt`
Components:
prompt_enhancer (`PreTrainedModel`) processor (`ProcessorMixin`) text_encoder (`PreTrainedModel`) tokenizer
(`PreTrainedTokenizerBase`) connectors (`LTX2TextConnectors`) duration_head (`LTX2DurationHead`) vae
(`AutoencoderKLLTX2Video`) video_processor (`VideoProcessor`) transformer (`LTX2VideoTransformer3DModel`)
scheduler (`FlowMatchEulerDiscreteScheduler`) audio_vae (`AutoencoderKLLTX2Audio`) guider (`LTX2Guidance`)
audio_guider (`LTX2Guidance`) diffusion_decoder (`LTX2VideoDiffusionDecoderModel`) vocoder (`LTX2Vocoder`)
Inputs:
prompt (`str`, *optional*):
The prompt or prompts to guide image generation.
conditions (`list`, *optional*):
`LTX2VideoCondition` (or list of them) placing image/video conditions at latent frame indices of the
generated video.
enable_prompt_enhancement (`bool`, *optional*, defaults to False):
Whether to run the prompt enhancer. Opt-in, matching the Lightricks reference pipelines.
system_prompt (`str`, *optional*):
System prompt for enhancement. Defaults to `LTX2_5_I2V_DEFAULT_SYSTEM_PROMPT` when a `PIL.Image.Image`
condition frame is available, else `LTX2_5_T2V_DEFAULT_SYSTEM_PROMPT`.
prompt_max_new_tokens (`int`, *optional*):
Maximum number of new tokens to generate during prompt enhancement. Defaults to 600, the LTX-2.5 Gemma-4
enhancer's budget.
prompt_enhancement_kwargs (`dict`, *optional*):
Keyword arguments for the enhancer's `.generate` call. Defaults to greedy decoding.
prompt_enhancement_seed (`int`, *optional*, defaults to 10):
Random seed for prompt enhancement (inert under LTX-2.5's greedy decoding).
generator (`Generator`, *optional*):
Torch generator for deterministic generation.
image (`Image | list`, *optional*):
Reference image(s) for denoising. Can be a single image or list of images.
negative_prompt (`str`, *optional*):
The prompt or prompts not to guide the image generation.
max_sequence_length (`int`, *optional*, defaults to 1024):
Maximum sequence length for prompt encoding.
min_seconds (`float`, *optional*, defaults to 1.0):
Lower bound on the auto-predicted duration.
max_seconds (`float`, *optional*, defaults to 20.0):
Upper bound on the auto-predicted duration. Must be strictly greater than `min_seconds`.
frame_rate (`float`, *optional*, defaults to 24.0):
Frames per second of the generated video.
height (`int`, *optional*, defaults to 512):
The height in pixels of the generated image.
width (`int`, *optional*, defaults to 704):
The width in pixels of the generated image.
image_crf (`int`, *optional*):
H.264 CRF used to re-compress the conditioning `image` before VAE encode, matching the compression the
model was trained against. `None` (default) resolves from the text-encoder generation (33 through
LTX-2.3, 18 for LTX-2.5). Pass `0` to skip re-compression. Requires a `PIL.Image.Image` when
re-compression runs.
num_frames (`int`, *optional*):
The number of frames in the generated video. Omit to auto-predict via the `duration_head` (see
`LTX2AutoDurationStep`).
reference_conditions (`list`, *optional*):
`LTX2ReferenceCondition` (or list of them) whose videos are encoded into extra latent tokens the IC-LoRA
adapter attends to.
reference_downscale_factor (`int`, *optional*, defaults to 1):
Ratio between the target and reference resolutions; 2 means the reference is preprocessed at half the
target resolution. Spatial coordinates are scaled by this factor so the reference tokens land in the
target coordinate space. Must match the factor the IC-LoRA was trained with.
conditioning_attention_strength (`float`, *optional*, defaults to 1.0):
Scalar in [0, 1] controlling how strongly the noisy tokens and reference tokens attend to each other. 1.0
(default) leaves attention unmasked.
conditioning_attention_mask (`Tensor`, *optional*):
Optional pixel-space mask of shape (1, 1, F, H, W) with values in [0, 1] giving spatially varying
attention strength. Downsampled to the reference's latent grid and multiplied by
`conditioning_attention_strength`.
num_videos_per_prompt (`int`, *optional*, defaults to 1):
The number of images to generate per prompt.
condition_latents (`list`, *optional*):
Per-condition normalized VAE latents of shape [1, C, F, H, W].
condition_strengths (`list`, *optional*):
Per-condition conditioning strengths.
condition_indices (`list`, *optional*):
Per-condition latent frame index at which the condition is applied.
condition_pixel_frames (`list`, *optional*):
Per-condition trimmed pixel frame count, used to clamp single-frame keyframe coords.
reference_latents (`Tensor`, *optional*):
Packed reference tokens of shape [1, total_reference_tokens, C], or `None` when no reference conditions
were supplied (`LTX2AutoReferenceEncoderStep` is skipped).
reference_coords (`Tensor`, *optional*):
RoPE coordinates for the reference tokens.
reference_token_counts (`list`, *optional*):
Per-reference token counts, in `reference_conditions` order.
latents (`Tensor`):
Pre-generated noisy latents for image generation.
noise_scale (`float`, *optional*):
Initial noise level for the un-conditioned tokens. `None` (default) resolves to `sigmas[0]` when custom
`sigmas` are supplied, else 1.0.
sigmas (`list`, *optional*):
Custom sigmas for the denoising process.
reference_cross_mask (`Tensor`, *optional*):
Per-reference-token noisy<->reference attention strengths of shape [1, num_ref_tokens].
num_inference_steps (`int`):
The number of denoising steps.
timesteps (`Tensor`):
Timesteps for the denoising process.
audio_latents (`Tensor`):
Optional pre-encoded audio latents; random noise is used when not provided.
**denoiser_input_fields (`None`, *optional*):
conditional model inputs for the denoiser: e.g. prompt_embeds, negative_prompt_embeds, etc.
use_cross_timestep (`bool`, *optional*, defaults to True):
Whether to condition the transformer on a separate per-token cross timestep (LTX-2.3+).
attention_kwargs (`dict`, *optional*):
Additional kwargs for attention processors.
image_latents (`Tensor`, *optional*):
VAE-encoded reference-image latents used for image-to-video conditioning.
output_type (`str`, *optional*, defaults to pil):
Output format: 'pil', 'np', 'pt'.
Outputs:
videos (`list`):
The generated videos.
audio (`Tensor`):
The generated audio waveform.
"""
model_name = "ltx2.5"
block_classes = [
LTX2AutoPromptEnhancerStep,
LTX2TextConditioningStep,
LTX2AutoDurationStep,
LTX2AutoVaeEncoderStep,
LTX2AutoConditionEncoderStep,
LTX2AutoReferenceEncoderStep,
LTX2AutoCoreDenoiseStep,
LTX25AutoDecoderStep,
]
block_names = [
"prompt_enhancer",
"text_encoder",
"duration",
"vae_encoder",
"condition_encoder",
"reference_encoder",
"denoise",
"decode",
]
# `num_frames` on `in_context` is a requirement, not a trigger: the in-context checkpoints ship without a
# `duration_head`, so the workflow drops `LTX2AutoDurationStep` and `LTX2ConditionEncoderStep` raises if
# `num_frames` is still `None`. Matches `LTX2InContextBlocks`, which omits the duration step outright.
_workflow_map = {
"text2video": {"prompt": True},
"image2video": {"image": True, "prompt": True},
"condition": {"conditions": True, "prompt": True},
"in_context": {"reference_conditions": True, "num_frames": True, "prompt": True},
}
@property
def description(self):
return (
"Auto blocks for LTX-2.5 supporting text-to-video, image-to-video, condition-to-video and in-context "
"(IC-LoRA) generation (joint video + audio). Identical to `LTX2AutoBlocks` except that the video decoder "
"is `LTX2DiffusionVaeDecoderStep`, since the diffusion decoder is the native default from LTX-2.5 on. To "
'decode with the convolutional VAE instead, swap the decode block: `blocks.sub_blocks["decode"] = '
"LTX2AutoDecoderStep()`."
)
@property
def outputs(self):
return [
OutputParam.template("videos"),
OutputParam("audio", type_hint=torch.Tensor, description="The generated audio waveform."),
]
|