dagloop5 commited on
Commit
7291620
·
verified ·
1 Parent(s): ef84ce5

Delete app.py

Browse files
Files changed (1) hide show
  1. app.py +0 -1006
app.py DELETED
@@ -1,1006 +0,0 @@
1
- import os
2
- import subprocess
3
- import sys
4
-
5
- # Disable torch.compile / dynamo before any torch import
6
- os.environ["TORCH_COMPILE_DISABLE"] = "1"
7
- os.environ["TORCHDYNAMO_DISABLE"] = "1"
8
-
9
-
10
- # Clone LTX-2 repo and install packages
11
- LTX_REPO_URL = "https://github.com/Lightricks/LTX-2.git"
12
- LTX_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "LTX-2")
13
-
14
- LTX_COMMIT = "ae855f8538843825f9015a419cf4ba5edaf5eec2" # known working commit with decode_video
15
-
16
- if not os.path.exists(LTX_REPO_DIR):
17
- print(f"Cloning {LTX_REPO_URL}...")
18
- subprocess.run(["git", "clone", LTX_REPO_URL, LTX_REPO_DIR], check=True)
19
- subprocess.run(["git", "checkout", LTX_COMMIT], cwd=LTX_REPO_DIR, check=True)
20
-
21
- print("Installing ltx-core and ltx-pipelines from cloned repo...")
22
- subprocess.run(
23
- [sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", "-e",
24
- os.path.join(LTX_REPO_DIR, "packages", "ltx-core"),
25
- "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines")],
26
- check=True,
27
- )
28
-
29
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
30
- sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
31
-
32
- import logging
33
- import random
34
- import tempfile
35
- from pathlib import Path
36
- import gc
37
- import hashlib
38
-
39
- import torch
40
- torch._dynamo.config.suppress_errors = True
41
- torch._dynamo.config.disable = True
42
-
43
- import spaces
44
- import gradio as gr
45
- import numpy as np
46
- from huggingface_hub import hf_hub_download, snapshot_download
47
- from safetensors.torch import load_file, save_file
48
- from safetensors import safe_open
49
- import json
50
- import requests
51
-
52
- from ltx_core.components.diffusion_steps import EulerDiffusionStep
53
- from ltx_core.components.noisers import GaussianNoiser
54
- from ltx_core.model.audio_vae import encode_audio as vae_encode_audio
55
- from ltx_core.model.upsampler import upsample_video
56
- from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number, decode_video as vae_decode_video
57
- from ltx_core.quantization import QuantizationPolicy
58
- from ltx_core.types import Audio, AudioLatentShape, VideoPixelShape
59
- from ltx_pipelines.distilled import DistilledPipeline
60
- from ltx_pipelines.utils import euler_denoising_loop
61
- from ltx_pipelines.utils.args import ImageConditioningInput
62
- from ltx_pipelines.utils.constants import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
63
- from ltx_pipelines.utils.helpers import (
64
- cleanup_memory,
65
- combined_image_conditionings,
66
- denoise_video_only,
67
- encode_prompts,
68
- simple_denoising_func,
69
- )
70
- from ltx_pipelines.utils.media_io import decode_audio_from_file, encode_video
71
- from ltx_core.loader.primitives import LoraPathStrengthAndSDOps
72
- from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP
73
-
74
- from ltx_core.model.transformer import attention as _attn_mod
75
-
76
- print(f"[ATTN] Before patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
77
- try:
78
- from xformers.ops import memory_efficient_attention as _mea
79
- from xformers.ops.fmha import cutlass
80
-
81
- def _cutlass_memory_efficient_attention(*args, **kwargs):
82
- # Force CUTLASS and avoid FlashAttention paths that are crashing.
83
- kwargs["op"] = (cutlass.FwOp, cutlass.BwOp)
84
- return _mea(*args, **kwargs)
85
-
86
- _attn_mod.memory_efficient_attention = _cutlass_memory_efficient_attention
87
- print(f"[ATTN] After patch: memory_efficient_attention={_attn_mod.memory_efficient_attention}")
88
- except Exception as e:
89
- print(f"[ATTN] xformers patch FAILED: {type(e).__name__}: {e}")
90
-
91
- logging.getLogger().setLevel(logging.INFO)
92
-
93
- MAX_SEED = np.iinfo(np.int32).max
94
- DEFAULT_PROMPT = (
95
- "An astronaut hatches from a fragile egg on the surface of the Moon, "
96
- "the shell cracking and peeling apart in gentle low-gravity motion. "
97
- "Fine lunar dust lifts and drifts outward with each movement, floating "
98
- "in slow arcs before settling back onto the ground."
99
- )
100
- DEFAULT_FRAME_RATE = 24.0
101
-
102
- # Resolution presets: (width, height)
103
- RESOLUTIONS = {
104
- "high": {"16:9": (1536, 1024), "9:16": (1024, 1536), "1:1": (1024, 1024)},
105
- "low": {"16:9": (768, 512), "9:16": (512, 768), "1:1": (768, 768)},
106
- }
107
-
108
-
109
- class LTX23DistilledA2VPipeline(DistilledPipeline):
110
- """DistilledPipeline with optional audio conditioning."""
111
-
112
- def __call__(
113
- self,
114
- prompt: str,
115
- seed: int,
116
- height: int,
117
- width: int,
118
- num_frames: int,
119
- frame_rate: float,
120
- images: list[ImageConditioningInput],
121
- audio_path: str | None = None,
122
- tiling_config: TilingConfig | None = None,
123
- enhance_prompt: bool = False,
124
- ):
125
- # Standard path when no audio input is provided.
126
- print(prompt)
127
- if audio_path is None:
128
- return super().__call__(
129
- prompt=prompt,
130
- seed=seed,
131
- height=height,
132
- width=width,
133
- num_frames=num_frames,
134
- frame_rate=frame_rate,
135
- images=images,
136
- tiling_config=tiling_config,
137
- enhance_prompt=enhance_prompt,
138
- )
139
-
140
- generator = torch.Generator(device=self.device).manual_seed(seed)
141
- noiser = GaussianNoiser(generator=generator)
142
- stepper = EulerDiffusionStep()
143
- dtype = torch.bfloat16
144
-
145
- (ctx_p,) = encode_prompts(
146
- [prompt],
147
- self.model_ledger,
148
- enhance_first_prompt=enhance_prompt,
149
- enhance_prompt_image=images[0].path if len(images) > 0 else None,
150
- )
151
- video_context, audio_context = ctx_p.video_encoding, ctx_p.audio_encoding
152
-
153
- video_duration = num_frames / frame_rate
154
- decoded_audio = decode_audio_from_file(audio_path, self.device, 0.0, video_duration)
155
- if decoded_audio is None:
156
- raise ValueError(f"Could not extract audio stream from {audio_path}")
157
-
158
- encoded_audio_latent = vae_encode_audio(decoded_audio, self.model_ledger.audio_encoder())
159
- audio_shape = AudioLatentShape.from_duration(batch=1, duration=video_duration, channels=8, mel_bins=16)
160
- expected_frames = audio_shape.frames
161
- actual_frames = encoded_audio_latent.shape[2]
162
-
163
- if actual_frames > expected_frames:
164
- encoded_audio_latent = encoded_audio_latent[:, :, :expected_frames, :]
165
- elif actual_frames < expected_frames:
166
- pad = torch.zeros(
167
- encoded_audio_latent.shape[0],
168
- encoded_audio_latent.shape[1],
169
- expected_frames - actual_frames,
170
- encoded_audio_latent.shape[3],
171
- device=encoded_audio_latent.device,
172
- dtype=encoded_audio_latent.dtype,
173
- )
174
- encoded_audio_latent = torch.cat([encoded_audio_latent, pad], dim=2)
175
-
176
- video_encoder = self.model_ledger.video_encoder()
177
- transformer = self.model_ledger.transformer()
178
- stage_1_sigmas = torch.tensor(DISTILLED_SIGMA_VALUES, device=self.device)
179
-
180
- def denoising_loop(sigmas, video_state, audio_state, stepper):
181
- return euler_denoising_loop(
182
- sigmas=sigmas,
183
- video_state=video_state,
184
- audio_state=audio_state,
185
- stepper=stepper,
186
- denoise_fn=simple_denoising_func(
187
- video_context=video_context,
188
- audio_context=audio_context,
189
- transformer=transformer,
190
- ),
191
- )
192
-
193
- stage_1_output_shape = VideoPixelShape(
194
- batch=1,
195
- frames=num_frames,
196
- width=width // 2,
197
- height=height // 2,
198
- fps=frame_rate,
199
- )
200
- stage_1_conditionings = combined_image_conditionings(
201
- images=images,
202
- height=stage_1_output_shape.height,
203
- width=stage_1_output_shape.width,
204
- video_encoder=video_encoder,
205
- dtype=dtype,
206
- device=self.device,
207
- )
208
- video_state = denoise_video_only(
209
- output_shape=stage_1_output_shape,
210
- conditionings=stage_1_conditionings,
211
- noiser=noiser,
212
- sigmas=stage_1_sigmas,
213
- stepper=stepper,
214
- denoising_loop_fn=denoising_loop,
215
- components=self.pipeline_components,
216
- dtype=dtype,
217
- device=self.device,
218
- initial_audio_latent=encoded_audio_latent,
219
- )
220
-
221
- torch.cuda.synchronize()
222
- cleanup_memory()
223
-
224
- upscaled_video_latent = upsample_video(
225
- latent=video_state.latent[:1],
226
- video_encoder=video_encoder,
227
- upsampler=self.model_ledger.spatial_upsampler(),
228
- )
229
- stage_2_sigmas = torch.tensor(STAGE_2_DISTILLED_SIGMA_VALUES, device=self.device)
230
- stage_2_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate)
231
- stage_2_conditionings = combined_image_conditionings(
232
- images=images,
233
- height=stage_2_output_shape.height,
234
- width=stage_2_output_shape.width,
235
- video_encoder=video_encoder,
236
- dtype=dtype,
237
- device=self.device,
238
- )
239
- video_state = denoise_video_only(
240
- output_shape=stage_2_output_shape,
241
- conditionings=stage_2_conditionings,
242
- noiser=noiser,
243
- sigmas=stage_2_sigmas,
244
- stepper=stepper,
245
- denoising_loop_fn=denoising_loop,
246
- components=self.pipeline_components,
247
- dtype=dtype,
248
- device=self.device,
249
- noise_scale=stage_2_sigmas[0],
250
- initial_video_latent=upscaled_video_latent,
251
- initial_audio_latent=encoded_audio_latent,
252
- )
253
-
254
- torch.cuda.synchronize()
255
- del transformer
256
- del video_encoder
257
- cleanup_memory()
258
-
259
- decoded_video = vae_decode_video(
260
- video_state.latent,
261
- self.model_ledger.video_decoder(),
262
- tiling_config,
263
- generator,
264
- )
265
- original_audio = Audio(
266
- waveform=decoded_audio.waveform.squeeze(0),
267
- sampling_rate=decoded_audio.sampling_rate,
268
- )
269
- return decoded_video, original_audio
270
-
271
-
272
- # Model repos
273
- LTX_MODEL_REPO = "Lightricks/LTX-2.3"
274
- GEMMA_REPO ="Lightricks/gemma-3-12b-it-qat-q4_0-unquantized"
275
- GEMMA_ABLITERATED_REPO = "Sikaworld1990/gemma-3-12b-it-abliterated-sikaworld-high-fidelity-edition-Ltx-2"
276
- GEMMA_ABLITERATED_FILE = "gemma-3-12b-it-abliterated-sikaworld-high-fidelity-edition.safetensors"
277
-
278
- # Download model checkpoints
279
- print("=" * 80)
280
- print("Downloading LTX-2.3 distilled model + Gemma...")
281
- print("=" * 80)
282
-
283
- # LoRA cache directory and currently-applied key
284
- LORA_CACHE_DIR = Path("lora_cache")
285
- LORA_CACHE_DIR.mkdir(exist_ok=True)
286
- current_lora_key: str | None = None
287
-
288
- PENDING_LORA_KEY: str | None = None
289
- PENDING_LORA_STATE: dict[str, torch.Tensor] | None = None
290
- PENDING_LORA_STATUS: str = "No LoRA state prepared yet."
291
-
292
- weights_dir = Path("weights")
293
- weights_dir.mkdir(exist_ok=True)
294
- checkpoint_path = hf_hub_download(
295
- repo_id="SulphurAI/Sulphur-2-base",
296
- filename="sulphur_distil_bf16.safetensors",
297
- local_dir=str(weights_dir),
298
- local_dir_use_symlinks=False,
299
- )
300
-
301
- print("[Gemma] Setting up abliterated Gemma text encoder...")
302
- MERGED_WEIGHTS = "/tmp/abliterated_gemma_merged.safetensors"
303
- gemma_root = "/tmp/abliterated_gemma"
304
- os.makedirs(gemma_root, exist_ok=True)
305
-
306
- gemma_official_dir = snapshot_download(
307
- repo_id=GEMMA_REPO,
308
- ignore_patterns=["*.safetensors", "*.safetensors.index.json"],
309
- )
310
-
311
- for fname in os.listdir(gemma_official_dir):
312
- src = os.path.join(gemma_official_dir, fname)
313
- dst = os.path.join(gemma_root, fname)
314
- if os.path.isfile(src) and not fname.endswith(".safetensors") and fname != "model.safetensors.index.json":
315
- if not os.path.exists(dst):
316
- os.symlink(src, dst)
317
-
318
- if os.path.exists(MERGED_WEIGHTS):
319
- print("[Gemma] Using cached merged weights")
320
- else:
321
- abliterated_weights_path = hf_hub_download(
322
- repo_id=GEMMA_ABLITERATED_REPO,
323
- filename=GEMMA_ABLITERATED_FILE,
324
- )
325
- index_path = hf_hub_download(
326
- repo_id=GEMMA_REPO,
327
- filename="model.safetensors.index.json"
328
- )
329
- with open(index_path) as f:
330
- weight_index = json.load(f)
331
-
332
- vision_keys = {}
333
- for key, shard in weight_index["weight_map"].items():
334
- if "vision_tower" in key or "multi_modal_projector" in key:
335
- vision_keys[key] = shard
336
- needed_shards = set(vision_keys.values())
337
-
338
- shard_paths = {}
339
- for shard_name in needed_shards:
340
- shard_paths[shard_name] = hf_hub_download(
341
- repo_id=GEMMA_REPO,
342
- filename=shard_name
343
- )
344
-
345
- _fp8_types = {torch.float8_e4m3fn, torch.float8_e5m2}
346
- raw = load_file(abliterated_weights_path)
347
- merged = {}
348
- for key, tensor in raw.items():
349
- t = tensor.to(torch.bfloat16) if tensor.dtype in _fp8_types else tensor
350
- merged[f"language_model.{key}"] = t
351
- del raw
352
-
353
- for key, shard_name in vision_keys.items():
354
- with safe_open(shard_paths[shard_name], framework="pt") as f:
355
- merged[key] = f.get_tensor(key)
356
-
357
- save_file(merged, MERGED_WEIGHTS)
358
- del merged
359
- gc.collect()
360
-
361
- weight_link = os.path.join(gemma_root, "model.safetensors")
362
- if os.path.exists(weight_link):
363
- os.remove(weight_link)
364
- os.symlink(MERGED_WEIGHTS, weight_link)
365
- print(f"[Gemma] Root ready: {gemma_root}")
366
-
367
- spatial_upsampler_path = hf_hub_download(repo_id=LTX_MODEL_REPO, filename="ltx-2.3-spatial-upscaler-x2-1.1.safetensors")
368
- gemma_root = snapshot_download(repo_id=GEMMA_REPO)
369
-
370
-
371
- # ---- Insert block (LoRA downloads) between lines 268 and 269 ----
372
- # LoRA repo + download the requested LoRA adapters
373
- LORA_REPO = "dagloop5/LoRA"
374
-
375
- print("=" * 80)
376
- print("Downloading LoRA adapters from dagloop5/LoRA...")
377
- print("=" * 80)
378
- pose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2_3_NSFW_furry_concat_v2.safetensors")
379
- general_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_reasoning_I2V_V3.safetensors")
380
- motion_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="motion_helper.safetensors")
381
- dreamlay_lora_path = hf_hub_download(repo_id="lynaNSFW/DR34ML4Y_AIO_NSFW_LTX23", filename="DR34ML4Y_LTXXX_V2.safetensors") # m15510n4ry, bl0wj0b, d0ubl3_bj, d0gg1e, c0wg1rl
382
- mself_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="Furry Hyper Masturbation - LTX-2 I2V v1.safetensors") # Hyperfap
383
- dramatic_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2.3 - Orgasm.safetensors") # "[He | She] is having am orgasm." (am or an?)
384
- fluid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_CREAMPIE_ANIMATION-V0.1.safetensors") # cum
385
- liquid_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="liquid_wet_dr1pp_ltx2_v1.0_scaled.safetensors") # wet dr1pp
386
- demopose_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="clapping-cheeks-audio-v001-alpha.safetensors")
387
- voice_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="hentai_voice_ltx23.safetensors")
388
- realism_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="FurryenhancerLTX2.3V4.094fused.safetensors")
389
- transition_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX-2_takerpov_lora_v1.2.safetensors") # takerpov1, taker pov
390
- physics_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_Physics_V2_000002000.safetensors")
391
- reasoning_lora_path = hf_hub_download(repo_id="LiconStudio/Ltx2.3-VBVR-lora-I2V", filename="Ltx2.3-Licon-VBVR-I2V-390K-R32.safetensors")
392
- twostep_lora_path = hf_hub_download(repo_id=LORA_REPO, filename="LTX2.3_Multi_step_video_reasoning_V0.1.safetensors")
393
-
394
- print(f"Pose LoRA: {pose_lora_path}")
395
- print(f"General LoRA: {general_lora_path}")
396
- print(f"Motion LoRA: {motion_lora_path}")
397
- print(f"Dreamlay LoRA: {dreamlay_lora_path}")
398
- print(f"Mself LoRA: {mself_lora_path}")
399
- print(f"Dramatic LoRA: {dramatic_lora_path}")
400
- print(f"Fluid LoRA: {fluid_lora_path}")
401
- print(f"Liquid LoRA: {liquid_lora_path}")
402
- print(f"Demopose LoRA: {demopose_lora_path}")
403
- print(f"Voice LoRA: {voice_lora_path}")
404
- print(f"Realism LoRA: {realism_lora_path}")
405
- print(f"Transition LoRA: {transition_lora_path}")
406
- print(f"Physics LoRA: {physics_lora_path}")
407
- print(f"Reasoning LoRA: {reasoning_lora_path}")
408
- print(f"Twostep LoRA: {twostep_lora_path}")
409
- # ----------------------------------------------------------------
410
-
411
- print(f"Checkpoint: {checkpoint_path}")
412
- print(f"Spatial upsampler: {spatial_upsampler_path}")
413
- print(f"[Gemma] Root ready: {gemma_root}")
414
-
415
- # Initialize pipeline WITH text encoder and optional audio support
416
- # ---- Replace block (pipeline init) lines 275-281 ----
417
- pipeline = LTX23DistilledA2VPipeline(
418
- distilled_checkpoint_path=checkpoint_path,
419
- spatial_upsampler_path=spatial_upsampler_path,
420
- gemma_root=gemma_root,
421
- loras=[],
422
- quantization=QuantizationPolicy.fp8_cast(), # keep FP8 quantization unchanged
423
- )
424
- # ----------------------------------------------------------------
425
-
426
- def _make_lora_key(pose_strength: float, general_strength: float, motion_strength: float, dreamlay_strength: float, mself_strength: float, dramatic_strength: float, fluid_strength: float, liquid_strength: float, demopose_strength: float, voice_strength: float, realism_strength: float, transition_strength: float, physics_strength: float, reasoning_strength: float, twostep_strength: float) -> tuple[str, str]:
427
- rp = round(float(pose_strength), 2)
428
- rg = round(float(general_strength), 2)
429
- rm = round(float(motion_strength), 2)
430
- rd = round(float(dreamlay_strength), 2)
431
- rs = round(float(mself_strength), 2)
432
- rr = round(float(dramatic_strength), 2)
433
- rf = round(float(fluid_strength), 2)
434
- rl = round(float(liquid_strength), 2)
435
- ro = round(float(demopose_strength), 2)
436
- rv = round(float(voice_strength), 2)
437
- re = round(float(realism_strength), 2)
438
- rt = round(float(transition_strength), 2)
439
- ry = round(float(physics_strength), 2)
440
- ri = round(float(reasoning_strength), 2)
441
- rw = round(float(twostep_strength), 2)
442
- key_str = f"{pose_lora_path}:{rp}|{general_lora_path}:{rg}|{motion_lora_path}:{rm}|{dreamlay_lora_path}:{rd}|{mself_lora_path}:{rs}|{dramatic_lora_path}:{rr}|{fluid_lora_path}:{rf}|{liquid_lora_path}:{rl}|{demopose_lora_path}:{ro}|{voice_lora_path}:{rv}|{realism_lora_path}:{re}|{transition_lora_path}:{rt}|{physics_lora_path}:{ry}|{reasoning_lora_path}:{ri}|{twostep_lora_path}:{rw}"
443
- key = hashlib.sha256(key_str.encode("utf-8")).hexdigest()
444
- return key, key_str
445
-
446
-
447
- def prepare_lora_cache(
448
- pose_strength: float,
449
- general_strength: float,
450
- motion_strength: float,
451
- dreamlay_strength: float,
452
- mself_strength: float,
453
- dramatic_strength: float,
454
- fluid_strength: float,
455
- liquid_strength: float,
456
- demopose_strength: float,
457
- voice_strength: float,
458
- realism_strength: float,
459
- transition_strength: float,
460
- physics_strength: float,
461
- reasoning_strength: float,
462
- twostep_strength: float,
463
- progress=gr.Progress(track_tqdm=True),
464
- ):
465
- """
466
- CPU-only step:
467
- - checks cache
468
- - loads cached fused transformer state_dict, or
469
- - builds fused transformer on CPU and saves it
470
- The resulting state_dict is stored in memory and can be applied later.
471
- """
472
- global PENDING_LORA_KEY, PENDING_LORA_STATE, PENDING_LORA_STATUS
473
-
474
- ledger = pipeline.model_ledger
475
- key, _ = _make_lora_key(pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength)
476
- cache_path = LORA_CACHE_DIR / f"{key}.safetensors"
477
-
478
- progress(0.05, desc="Preparing LoRA state")
479
- if cache_path.exists():
480
- try:
481
- progress(0.20, desc="Loading cached fused state")
482
- state = load_file(str(cache_path))
483
- PENDING_LORA_KEY = key
484
- PENDING_LORA_STATE = state
485
- PENDING_LORA_STATUS = f"Loaded cached LoRA state: {cache_path.name}"
486
- return PENDING_LORA_STATUS
487
- except Exception as e:
488
- print(f"[LoRA] Cache load failed: {type(e).__name__}: {e}")
489
-
490
- entries = [
491
- (pose_lora_path, round(float(pose_strength), 2)),
492
- (general_lora_path, round(float(general_strength), 2)),
493
- (motion_lora_path, round(float(motion_strength), 2)),
494
- (dreamlay_lora_path, round(float(dreamlay_strength), 2)),
495
- (mself_lora_path, round(float(mself_strength), 2)),
496
- (dramatic_lora_path, round(float(dramatic_strength), 2)),
497
- (fluid_lora_path, round(float(fluid_strength), 2)),
498
- (liquid_lora_path, round(float(liquid_strength), 2)),
499
- (demopose_lora_path, round(float(demopose_strength), 2)),
500
- (voice_lora_path, round(float(voice_strength), 2)),
501
- (realism_lora_path, round(float(realism_strength), 2)),
502
- (transition_lora_path, round(float(transition_strength), 2)),
503
- (physics_lora_path, round(float(physics_strength), 2)),
504
- (reasoning_lora_path, round(float(reasoning_strength), 2)),
505
- (twostep_lora_path, round(float(twostep_strength), 2)),
506
- ]
507
- loras_for_builder = [
508
- LoraPathStrengthAndSDOps(path, strength, LTXV_LORA_COMFY_RENAMING_MAP)
509
- for path, strength in entries
510
- if path is not None and float(strength) != 0.0
511
- ]
512
-
513
- if not loras_for_builder:
514
- PENDING_LORA_KEY = None
515
- PENDING_LORA_STATE = None
516
- PENDING_LORA_STATUS = "No non-zero LoRA strengths selected; nothing to prepare."
517
- return PENDING_LORA_STATUS
518
-
519
- tmp_ledger = None
520
- new_transformer_cpu = None
521
- try:
522
- progress(0.35, desc="Building fused CPU transformer")
523
- tmp_ledger = pipeline.model_ledger.__class__(
524
- dtype=ledger.dtype,
525
- device=torch.device("cpu"),
526
- checkpoint_path=str(checkpoint_path),
527
- spatial_upsampler_path=str(spatial_upsampler_path),
528
- gemma_root_path=str(gemma_root),
529
- loras=tuple(loras_for_builder),
530
- quantization=getattr(ledger, "quantization", None),
531
- )
532
- new_transformer_cpu = tmp_ledger.transformer()
533
-
534
- progress(0.70, desc="Extracting fused state_dict")
535
- state = {
536
- k: v.detach().cpu().contiguous()
537
- for k, v in new_transformer_cpu.state_dict().items()
538
- }
539
- save_file(state, str(cache_path))
540
-
541
- PENDING_LORA_KEY = key
542
- PENDING_LORA_STATE = state
543
- PENDING_LORA_STATUS = f"Built and cached LoRA state: {cache_path.name}"
544
- return PENDING_LORA_STATUS
545
-
546
- except Exception as e:
547
- import traceback
548
- print(f"[LoRA] Prepare failed: {type(e).__name__}: {e}")
549
- print(traceback.format_exc())
550
- PENDING_LORA_KEY = None
551
- PENDING_LORA_STATE = None
552
- PENDING_LORA_STATUS = f"LoRA prepare failed: {type(e).__name__}: {e}"
553
- return PENDING_LORA_STATUS
554
-
555
- finally:
556
- try:
557
- del new_transformer_cpu
558
- except Exception:
559
- pass
560
- try:
561
- del tmp_ledger
562
- except Exception:
563
- pass
564
- gc.collect()
565
-
566
-
567
- def apply_prepared_lora_state_to_pipeline():
568
- """
569
- Fast step: copy the already prepared CPU state into the live transformer.
570
- This is the only part that should remain near generation time.
571
- """
572
- global current_lora_key, PENDING_LORA_KEY, PENDING_LORA_STATE
573
-
574
- if PENDING_LORA_STATE is None or PENDING_LORA_KEY is None:
575
- print("[LoRA] No prepared LoRA state available; skipping.")
576
- return False
577
-
578
- if current_lora_key == PENDING_LORA_KEY:
579
- print("[LoRA] Prepared LoRA state already active; skipping.")
580
- return True
581
-
582
- existing_transformer = _transformer
583
- with torch.no_grad():
584
- missing, unexpected = existing_transformer.load_state_dict(PENDING_LORA_STATE, strict=False)
585
- if missing or unexpected:
586
- print(f"[LoRA] load_state_dict mismatch: missing={len(missing)}, unexpected={len(unexpected)}")
587
-
588
- current_lora_key = PENDING_LORA_KEY
589
- print("[LoRA] Prepared LoRA state applied to the pipeline.")
590
- return True
591
-
592
- # ---- REPLACE PRELOAD BLOCK START ----
593
- # Preload all models for ZeroGPU tensor packing.
594
- print("Preloading all models (including Gemma and audio components)...")
595
- ledger = pipeline.model_ledger
596
-
597
- # Save the original factory methods so we can rebuild individual components later.
598
- # These are bound callables on ledger that will call the builder when invoked.
599
- _orig_transformer_factory = ledger.transformer
600
- _orig_video_encoder_factory = ledger.video_encoder
601
- _orig_video_decoder_factory = ledger.video_decoder
602
- _orig_audio_encoder_factory = ledger.audio_encoder
603
- _orig_audio_decoder_factory = ledger.audio_decoder
604
- _orig_vocoder_factory = ledger.vocoder
605
- _orig_spatial_upsampler_factory = ledger.spatial_upsampler
606
- _orig_text_encoder_factory = ledger.text_encoder
607
- _orig_gemma_embeddings_factory = ledger.gemma_embeddings_processor
608
-
609
- # Call the original factories once to create the cached instances we will serve by default.
610
- _transformer = _orig_transformer_factory()
611
- _video_encoder = _orig_video_encoder_factory()
612
- _video_decoder = _orig_video_decoder_factory()
613
- _audio_encoder = _orig_audio_encoder_factory()
614
- _audio_decoder = _orig_audio_decoder_factory()
615
- _vocoder = _orig_vocoder_factory()
616
- _spatial_upsampler = _orig_spatial_upsampler_factory()
617
- _text_encoder = _orig_text_encoder_factory()
618
- _embeddings_processor = _orig_gemma_embeddings_factory()
619
-
620
- # Replace ledger methods with lightweight lambdas that return the cached instances.
621
- # We keep the original factories above so we can call them later to rebuild components.
622
- ledger.transformer = lambda: _transformer
623
- ledger.video_encoder = lambda: _video_encoder
624
- ledger.video_decoder = lambda: _video_decoder
625
- ledger.audio_encoder = lambda: _audio_encoder
626
- ledger.audio_decoder = lambda: _audio_decoder
627
- ledger.vocoder = lambda: _vocoder
628
- ledger.spatial_upsampler = lambda: _spatial_upsampler
629
- ledger.text_encoder = lambda: _text_encoder
630
- ledger.gemma_embeddings_processor = lambda: _embeddings_processor
631
-
632
- print("All models preloaded (including Gemma text encoder and audio encoder)!")
633
- # ---- REPLACE PRELOAD BLOCK END ----
634
-
635
- print("=" * 80)
636
- print("Pipeline ready!")
637
- print("=" * 80)
638
-
639
-
640
- def log_memory(tag: str):
641
- if torch.cuda.is_available():
642
- allocated = torch.cuda.memory_allocated() / 1024**3
643
- peak = torch.cuda.max_memory_allocated() / 1024**3
644
- free, total = torch.cuda.mem_get_info()
645
- print(f"[VRAM {tag}] allocated={allocated:.2f}GB peak={peak:.2f}GB free={free / 1024**3:.2f}GB total={total / 1024**3:.2f}GB")
646
-
647
-
648
- def detect_aspect_ratio(image) -> str:
649
- if image is None:
650
- return "16:9"
651
- if hasattr(image, "size"):
652
- w, h = image.size
653
- elif hasattr(image, "shape"):
654
- h, w = image.shape[:2]
655
- else:
656
- return "16:9"
657
- ratio = w / h
658
- candidates = {"16:9": 16 / 9, "9:16": 9 / 16, "1:1": 1.0}
659
- return min(candidates, key=lambda k: abs(ratio - candidates[k]))
660
-
661
-
662
- def on_image_upload(first_image, last_image, high_res):
663
- ref_image = first_image if first_image is not None else last_image
664
- aspect = detect_aspect_ratio(ref_image)
665
- tier = "high" if high_res else "low"
666
- w, h = RESOLUTIONS[tier][aspect]
667
- return gr.update(value=w), gr.update(value=h)
668
-
669
-
670
- def on_highres_toggle(first_image, last_image, high_res):
671
- ref_image = first_image if first_image is not None else last_image
672
- aspect = detect_aspect_ratio(ref_image)
673
- tier = "high" if high_res else "low"
674
- w, h = RESOLUTIONS[tier][aspect]
675
- return gr.update(value=w), gr.update(value=h)
676
-
677
-
678
- def get_gpu_duration(
679
- first_image,
680
- last_image,
681
- input_audio,
682
- prompt: str,
683
- duration: float = 0.0,
684
- gpu_duration: float = 0.0,
685
- enhance_prompt: bool = False,
686
- seed: int = 42,
687
- randomize_seed: bool = False,
688
- height: int = 0.0,
689
- width: int = 0.0,
690
- pose_strength: float = 0.0,
691
- general_strength: float = 0.0,
692
- motion_strength: float = 0.0,
693
- dreamlay_strength: float = 0.0,
694
- mself_strength: float = 0.0,
695
- dramatic_strength: float = 0.0,
696
- fluid_strength: float = 0.0,
697
- liquid_strength: float = 0.0,
698
- demopose_strength: float = 0.0,
699
- voice_strength: float = 0.0,
700
- realism_strength: float = 0.0,
701
- transition_strength: float = 0.0,
702
- physics_strength: float = 0.0,
703
- reasoning_strength: float = 0.0,
704
- twostep_strength: float = 0.0,
705
- progress=None,
706
- ):
707
- return int(gpu_duration)
708
-
709
- @spaces.GPU(duration=get_gpu_duration)
710
- @torch.inference_mode()
711
- def generate_video(
712
- first_image,
713
- last_image,
714
- input_audio,
715
- prompt: str,
716
- duration: float = 0.0,
717
- gpu_duration: float = 0.0,
718
- enhance_prompt: bool = True,
719
- seed: int = 42,
720
- randomize_seed: bool = True,
721
- height: int = 0.0,
722
- width: int = 0.0,
723
- pose_strength: float = 0.0,
724
- general_strength: float = 0.0,
725
- motion_strength: float = 0.0,
726
- dreamlay_strength: float = 0.0,
727
- mself_strength: float = 0.0,
728
- dramatic_strength: float = 0.0,
729
- fluid_strength: float = 0.0,
730
- liquid_strength: float = 0.0,
731
- demopose_strength: float = 0.0,
732
- voice_strength: float = 0.0,
733
- realism_strength: float = 0.0,
734
- transition_strength: float = 0.0,
735
- physics_strength: float = 0.0,
736
- reasoning_strength: float = 0.0,
737
- twostep_strength: float = 0.0,
738
- progress=gr.Progress(track_tqdm=True),
739
- ):
740
- try:
741
- torch.cuda.reset_peak_memory_stats()
742
- log_memory("start")
743
-
744
- current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
745
-
746
- frame_rate = DEFAULT_FRAME_RATE
747
- num_frames = int(duration * frame_rate) + 1
748
- num_frames = ((num_frames - 1 + 7) // 8) * 8 + 1
749
-
750
- print(f"Generating: {height}x{width}, {num_frames} frames ({duration}s), seed={current_seed}")
751
-
752
- images = []
753
- output_dir = Path("outputs")
754
- output_dir.mkdir(exist_ok=True)
755
-
756
- if first_image is not None:
757
- temp_first_path = output_dir / f"temp_first_{current_seed}.jpg"
758
- if hasattr(first_image, "save"):
759
- first_image.save(temp_first_path)
760
- else:
761
- temp_first_path = Path(first_image)
762
- images.append(ImageConditioningInput(path=str(temp_first_path), frame_idx=0, strength=1.0))
763
-
764
- if last_image is not None:
765
- temp_last_path = output_dir / f"temp_last_{current_seed}.jpg"
766
- if hasattr(last_image, "save"):
767
- last_image.save(temp_last_path)
768
- else:
769
- temp_last_path = Path(last_image)
770
- images.append(ImageConditioningInput(path=str(temp_last_path), frame_idx=num_frames - 1, strength=1.0))
771
-
772
- tiling_config = TilingConfig.default()
773
- video_chunks_number = get_video_chunks_number(num_frames, tiling_config)
774
-
775
- log_memory("before pipeline call")
776
-
777
- apply_prepared_lora_state_to_pipeline()
778
-
779
- video, audio = pipeline(
780
- prompt=prompt,
781
- seed=current_seed,
782
- height=int(height),
783
- width=int(width),
784
- num_frames=num_frames,
785
- frame_rate=frame_rate,
786
- images=images,
787
- audio_path=input_audio,
788
- tiling_config=tiling_config,
789
- enhance_prompt=enhance_prompt,
790
- )
791
-
792
- log_memory("after pipeline call")
793
-
794
- output_path = tempfile.mktemp(suffix=".mp4")
795
- encode_video(
796
- video=video,
797
- fps=frame_rate,
798
- audio=audio,
799
- output_path=output_path,
800
- video_chunks_number=video_chunks_number,
801
- )
802
-
803
- log_memory("after encode_video")
804
- return str(output_path), current_seed
805
-
806
- except Exception as e:
807
- import traceback
808
- log_memory("on error")
809
- print(f"Error: {str(e)}\n{traceback.format_exc()}")
810
- return None, current_seed
811
-
812
-
813
- with gr.Blocks(title="LTX-2.3 Distilled") as demo:
814
- gr.Markdown("# LTX-2.3 F2LF with Fast Audio-Video Generation with Frame Conditioning")
815
-
816
-
817
- with gr.Row():
818
- with gr.Column():
819
- with gr.Row():
820
- first_image = gr.Image(label="First Frame (Optional)", type="pil")
821
- last_image = gr.Image(label="Last Frame (Optional)", type="pil")
822
- input_audio = gr.Audio(label="Audio Input (Optional)", type="filepath")
823
- prompt = gr.Textbox(
824
- label="Prompt",
825
- info="for best results - make it as elaborate as possible",
826
- value="Make this image come alive with cinematic motion, smooth animation",
827
- lines=3,
828
- placeholder="Describe the motion and animation you want...",
829
- )
830
- duration = gr.Slider(label="Duration (seconds)", minimum=1.0, maximum=30.0, value=10.0, step=0.1)
831
-
832
-
833
- generate_btn = gr.Button("Generate Video", variant="primary", size="lg")
834
-
835
- with gr.Accordion("Advanced Settings", open=False):
836
- seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, value=10, step=1)
837
- randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
838
- with gr.Row():
839
- width = gr.Number(label="Width", value=1536, precision=0)
840
- height = gr.Number(label="Height", value=1024, precision=0)
841
- with gr.Row():
842
- enhance_prompt = gr.Checkbox(label="Enhance Prompt", value=False)
843
- high_res = gr.Checkbox(label="High Resolution", value=True)
844
- with gr.Column():
845
- gr.Markdown("### LoRA adapter strengths (set to 0 to disable; slow and WIP)")
846
- pose_strength = gr.Slider(
847
- label="Anthro Enhancer strength",
848
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
849
- )
850
- general_strength = gr.Slider(
851
- label="Reasoning Enhancer strength",
852
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
853
- )
854
- motion_strength = gr.Slider(
855
- label="Anthro Posing Helper strength",
856
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
857
- )
858
- dreamlay_strength = gr.Slider(
859
- label="Dreamlay strength",
860
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
861
- )
862
- mself_strength = gr.Slider(
863
- label="Mself strength",
864
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
865
- )
866
- dramatic_strength = gr.Slider(
867
- label="Dramatic strength",
868
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
869
- )
870
- fluid_strength = gr.Slider(
871
- label="Fluid Helper strength",
872
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
873
- )
874
- liquid_strength = gr.Slider(
875
- label="Liquid Helper strength",
876
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
877
- )
878
- demopose_strength = gr.Slider(
879
- label="Audio Helper strength",
880
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
881
- )
882
- voice_strength = gr.Slider(
883
- label="Voice Helper strength",
884
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
885
- )
886
- realism_strength = gr.Slider(
887
- label="Anthro Realism strength",
888
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
889
- )
890
- transition_strength = gr.Slider(
891
- label="POV strength",
892
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
893
- )
894
- physics_strength = gr.Slider(
895
- label="Physics strength",
896
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
897
- )
898
- reasoning_strength = gr.Slider(
899
- label="Official Reasoning strength",
900
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
901
- )
902
- twostep_strength = gr.Slider(
903
- label="Two Step Reasoning strength",
904
- minimum=0.0, maximum=2.0, value=0.0, step=0.01
905
- )
906
- prepare_lora_btn = gr.Button("Prepare / Load LoRA Cache", variant="secondary")
907
- lora_status = gr.Textbox(
908
- label="LoRA Cache Status",
909
- value="No LoRA state prepared yet.",
910
- interactive=False,
911
- )
912
-
913
- with gr.Column():
914
- output_video = gr.Video(label="Generated Video", autoplay=False)
915
- gpu_duration = gr.Slider(
916
- label="ZeroGPU duration (seconds; 10 second Img2Vid with 1024x1024 and LoRAs = ~70)",
917
- minimum=30.0,
918
- maximum=240.0,
919
- value=75.0,
920
- step=1.0,
921
- )
922
-
923
- gr.Examples(
924
- examples=[
925
- [
926
- None,
927
- "pinkknit.jpg",
928
- None,
929
- "The camera falls downward through darkness as if dropped into a tunnel. "
930
- "As it slows, five friends wearing pink knitted hats and sunglasses lean "
931
- "over and look down toward the camera with curious expressions. The lens "
932
- "has a strong fisheye effect, creating a circular frame around them. They "
933
- "crowd together closely, forming a symmetrical cluster while staring "
934
- "directly into the lens.",
935
- 3.0,
936
- 80.0,
937
- False,
938
- 42,
939
- True,
940
- 1024,
941
- 1024,
942
- 0.0, # pose_strength (example)
943
- 0.0, # general_strength (example)
944
- 0.0, # motion_strength (example)
945
- 0.0,
946
- 0.0,
947
- 0.0,
948
- 0.0,
949
- 0.0,
950
- 0.0,
951
- 0.0,
952
- 0.0,
953
- 0.0,
954
- 0.0,
955
- 0.0,
956
- 0.0,
957
- ],
958
- ],
959
- inputs=[
960
- first_image, last_image, input_audio, prompt, duration, gpu_duration,
961
- enhance_prompt, seed, randomize_seed, height, width,
962
- pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength,
963
- ],
964
- )
965
-
966
- first_image.change(
967
- fn=on_image_upload,
968
- inputs=[first_image, last_image, high_res],
969
- outputs=[width, height],
970
- )
971
-
972
- last_image.change(
973
- fn=on_image_upload,
974
- inputs=[first_image, last_image, high_res],
975
- outputs=[width, height],
976
- )
977
-
978
- high_res.change(
979
- fn=on_highres_toggle,
980
- inputs=[first_image, last_image, high_res],
981
- outputs=[width, height],
982
- )
983
-
984
- prepare_lora_btn.click(
985
- fn=prepare_lora_cache,
986
- inputs=[pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength],
987
- outputs=[lora_status],
988
- )
989
-
990
- generate_btn.click(
991
- fn=generate_video,
992
- inputs=[
993
- first_image, last_image, input_audio, prompt, duration, gpu_duration, enhance_prompt,
994
- seed, randomize_seed, height, width,
995
- pose_strength, general_strength, motion_strength, dreamlay_strength, mself_strength, dramatic_strength, fluid_strength, liquid_strength, demopose_strength, voice_strength, realism_strength, transition_strength, physics_strength, reasoning_strength, twostep_strength,
996
- ],
997
- outputs=[output_video, seed],
998
- )
999
-
1000
-
1001
- css = """
1002
- .fillable{max-width: 1200px !important}
1003
- """
1004
-
1005
- if __name__ == "__main__":
1006
- demo.launch(theme=gr.themes.Citrus(), css=css)